Migrate proposals

hypha.apply.funds.management.commands.migrate_proposals

Command

Bases: MigrateCommand

help class-attribute instance-attribute

help = 'Application migration script. Requires a source JSON file.'

data class-attribute instance-attribute

data = []

terms class-attribute instance-attribute

terms = {}

patched_status_field class-attribute instance-attribute

patched_status_field = FSMField(default=INITIAL_STATE, protected=False)

CONTENT_TYPE class-attribute instance-attribute

CONTENT_TYPE = 'fund'

FUND_NAME class-attribute instance-attribute

FUND_NAME = 'Internet Freedom Fund (archive fund)'

ROUND_NAME class-attribute instance-attribute

ROUND_NAME = 'Internet Freedom Fund (archive round)'

APPLICATION_TYPE class-attribute instance-attribute

APPLICATION_TYPE = 'proposal'

STREAMFIELD_MAP class-attribute instance-attribute

STREAMFIELD_MAP = {'title': {'id': 'title', 'type': 'direct'}, 'field_proposal_funding': {'id': 'value', 'type': 'value'}, 'field_proposal_term_time': {'id': 'duration', 'type': 'value'}, 'field_proposal_legal_entity_name': {'id': '739a413b-46cc-4936-82ce-e68c2dfa41ca', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_common_name': {'id': 'full_name', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_entity_contact': {'id': 'a3c9af86-d047-4663-864a-b6dd97a60c39', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_entity_mail': {'id': 'email', 'type': 'value', 'key': 'email'}, 'field_proposal_entity_phone': {'id': '40479d2a-7d53-4c81-834a-775ccd6c91c0', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_entity_address': {'id': 'f7e431b1-9965-4ebe-ab30-a00ff4b972ec', 'type': 'address', 'map': {'administrative_area': 'administrativearea', 'country': 'country', 'locality': 'localityname', 'postal_code': 'postalcode', 'thoroughfare': 'thoroughfare', 'premise': 'premise'}}, 'field_proposal_summary': {'id': 'a7502e97-5f2e-417f-b08c-588d367e40e5', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_narrative': {'id': '072f181b-90a2-4bb2-986d-55e1aaa9f348', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_objectives': {'id': 'a7ae7375-4569-47e2-8ee7-3c3d441375a9', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_activities': {'id': '50328cc9-879d-4817-8454-2062ac47aef9', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_budget_details': {'id': '7b0b0af4-009f-45db-b20c-5f991bce7752', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_similar_efforts': {'id': 'ba3733f3-bee8-4fe8-bdde-36812aa4df77', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_evaluation': {'id': '23028eab-92c3-4c30-9a3e-5604dec0854d', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_sustainability': {'id': '88d635e4-81d6-413c-8e09-52b74015e78b', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_fund_info': {'id': '38072bb7-fcdd-4f74-9bfb-db45bfeb07a7', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_background': {'id': 'f8b340d0-8c0c-41f8-acb5-662c676e2bbd', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_references': {'id': '8b2572ce-d118-41c4-b5d7-59f4ffe44431', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_community': {'id': '1b93fcd1-c6cd-432a-b831-a0fb680e327e', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_status': {'id': '036fa233-c42a-4fc6-861a-ff40450efc7d', 'type': 'category', 'key': 'tid'}, 'field_type': {'id': '7d69aeec-009d-4de2-8dd2-6b0aacb4578f', 'type': 'category', 'key': 'tid'}, 'field_proposal_focus': {'id': '328590d3-fefd-410d-b745-12f2efdd5437', 'type': 'category', 'key': 'tid'}, 'field_proposal_beneficiaries': {'id': 'f18f0399-538b-4bf3-9cd5-4457138814a8', 'type': 'category', 'key': 'tid'}, 'field_proposal_theme': {'id': 'a9b2b6de-fb7b-4709-aa59-f0ad987a677a', 'type': 'category', 'key': 'tid'}, 'field_technology_attribute': {'id': '251cf41a-0a49-4725-8d5a-5e496d018647', 'type': 'category', 'key': 'tid'}, 'field_term_region': {'id': 'ed6244ae-6903-4412-8b7a-c219ed25dfbb', 'type': 'category', 'key': 'tid'}, 'field_term_country': {'id': '4b79c527-bf24-47f1-87a7-39945e70caeb', 'type': 'category', 'key': 'tid'}, 'field_proposal_upload': {'id': '6bec61a1-3527-4e21-aa65-f26d845bbb68', 'type': 'file'}, 'field_proposal_comments': {'id': 'c0ff8444-8d43-46ef-8498-ed1a32c09c6a', 'type': 'value', 'key': 'safe_value'}}

add_arguments

add_arguments(parser)

Source code in hypha/apply/funds/management/commands/migration_base.py

def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle

handle(*args, **options)

Source code in hypha/apply/funds/management/commands/migration_base.py

@transaction.atomic
def handle(self, *args, **options):
    # Prepare the list of categories.
    for item in CATEGORIES:
        category, _ = Category.objects.get_or_create(name=item["category"])
        option, _ = Option.objects.get_or_create(
            value=item["name"], category=category
        )
        self.terms[item["tid"]] = option

    with options["source"] as json_data:
        self.data = json.load(json_data)

        counter = 0
        for id in self.data:
            self.process(id)
            counter += 1

        self.stdout.write(f"Imported {counter} submissions.")

process

process(id)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process(self, id):
    node = self.data[id]

    try:
        submission = ApplicationSubmission.objects.get(drupal_id=node["nid"])
    except ApplicationSubmission.DoesNotExist:
        submission = ApplicationSubmission(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in submission._meta.local_fields:
        if field.name == "submit_time":
            field.auto_now_add = False

    # TODO timezone?
    submission.submit_time = datetime.fromtimestamp(
        int(node["created"]), timezone.utc
    )
    submission.user = self.get_user(node["uid"])

    if self.CONTENT_TYPE == "fund":
        FUND = FundType.objects.get(title=self.FUND_NAME)
        submission.page = FUND
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        submission.round = ROUND
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[1]
        submission.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        submission.page = LAB
        FORM = LabBaseForm.objects.get(lab=LAB)
        submission.form_fields = FORM.form.form_fields

    submission.status = self.get_workflow_state(node)

    if "proposal_nid" in node:
        try:
            submission.next = ApplicationSubmission.objects.get(
                drupal_id=node["proposal_nid"]
            )
        except ApplicationSubmission.DoesNotExist:
            self.stdout.write(
                "No related proposal found, please import proposals before applications."
            )
            pass

    form_data = {
        "skip_account_creation_notification": True,
    }

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "value" not in form_data or not form_data["value"]:
        form_data["value"] = 0

    if "duration" not in form_data or not form_data["duration"]:
        form_data["duration"] = "1"

    if "email" not in form_data or not form_data["email"]:
        if hasattr(submission.user, "email"):
            form_data["email"] = submission.user.email
        else:
            form_data["email"] = f"user+{node['uid']}@example.com"

    submission.form_data = form_data

    try:
        submission.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user

get_user(uid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[
                node["field_application_request"]["target_id"]
            ]
            values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "address" and "map" in mapping:
        try:
            value_map = mapping["map"]
            value = {}
            for item in value_map:
                value[value_map[item]] = source_value[item]
            value = json.dumps(value)
        except TypeError:
            value = {}
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False
    elif mapping_type == "category":
        if not source_value:
            value = []
        else:
            if isinstance(source_value, dict):
                option = self.get_referenced_term(source_value[key])
                value = [option] if option else []
            else:
                value = []
                for item in source_value:
                    option = self.get_referenced_term(item[key])
                    if option:
                        value.append(option)
    elif mapping_type == "file":
        value = self.process_file(source_value)

    return value

get_referenced_term

get_referenced_term(tid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_term(self, tid):
    try:
        term = self.terms[tid]
        return term.id
    except KeyError:
        return None

get_referenced_node

get_referenced_node(nid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_node(self, nid):
    pass

get_workflow_state

get_workflow_state(node)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_workflow_state(self, node):
    """
    workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}}
    """
    states_request = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "internal_review",
        "ready_for_reply": "post_review_discussion",
        "contract_review": "post_review_discussion",
        "in_contract": "accepted",
        "invited_for_proposal": "accepted",
        "dropped_concept_note": "rejected",
        "dropped": "rejected",
        "dropped_without_review": "rejected",
    }

    states_concept = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "concept_internal_review",
        "ready_for_reply": "concept_review_discussion",
        "contract_review": "concept_review_discussion",
        "in_contract": "invited_to_proposal",
        "invited_for_proposal": "invited_to_proposal",
        "dropped_concept_note": "concept_rejected",
        "dropped": "concept_rejected",
        "dropped_without_review": "concept_rejected",
    }

    states_proposal = {
        "draft": "draft_proposal",
        "published": "proposal_discussion",
        "in_discussion": "proposal_discussion",
        "council_review": "external_review",
        "ready_for_reply": "proposal_more_info",
        "contract_review": "post_external_review_discussion",
        "in_contract": "proposal_accepted",
        "invited_for_proposal": "proposal_accepted",
        "dropped_concept_note": "proposal_rejected",
        "dropped": "proposal_rejected",
        "dropped_without_review": "proposal_rejected",
    }

    if self.APPLICATION_TYPE == "request":
        workflow_state = states_request.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "concept":
        workflow_state = states_concept.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "proposal":
        workflow_state = states_proposal.get(
            node["workbench_moderation"]["current"]["state"], "draft_proposal"
        )
    else:
        workflow_state = None

    return workflow_state

nl2br

nl2br(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")

process_file

process_file(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process_file(self, value):
    if isinstance(value, dict):
        value = [value]

    files = []

    for file_data in value:
        parts = urlsplit(file_data["uri"])
        file_path = os.path.join(
            "files", "private", parts.netloc, *parts.path.split("/")
        )
        saved_file = migration_storage.open(file_path)
        saved_file.name = file_data["filename"]
        files.append(saved_file)

    return files