Migrate fellowship application

hypha.apply.funds.management.commands.migrate_fellowship_application

Command

Bases: MigrateCommand

help class-attribute instance-attribute

help = 'Application migration script. Requires a source JSON file.'

data class-attribute instance-attribute

data = []

terms class-attribute instance-attribute

terms = {}

patched_status_field class-attribute instance-attribute

patched_status_field = FSMField(default=INITIAL_STATE, protected=False)

CONTENT_TYPE class-attribute instance-attribute

CONTENT_TYPE = 'fund'

FUND_NAME class-attribute instance-attribute

FUND_NAME = 'Fellowship (archive fund)'

ROUND_NAME class-attribute instance-attribute

ROUND_NAME = 'Fellowship (archive round)'

APPLICATION_TYPE class-attribute instance-attribute

APPLICATION_TYPE = 'concept'

STREAMFIELD_MAP class-attribute instance-attribute

STREAMFIELD_MAP = {'title': {'id': 'title', 'type': 'direct'}, 'field_application_name': {'id': 'full_name', 'type': 'value', 'key': 'safe_value'}, 'field_application_mail': {'id': 'email', 'type': 'value', 'key': 'email'}, 'field_application_position': {'id': '1282223d-77f5-4047-be03-4df4c4b2148a', 'type': 'value', 'key': 'safe_value'}, 'field_application_role': {'id': '9c0256e4-42e1-41fe-9880-7f621d6c3458', 'type': 'value', 'key': 'safe_value'}, 'field_application_preapplied': {'id': 'f8efef0a-0632-4c81-b4db-7bc6a06caa7d', 'type': 'map', 'map': {'0': 'No', '1': 'Yes'}}, 'field_application_describe': {'id': '1eb8b4e3-e2bb-4810-a8ce-3fc82a3192c8', 'type': 'value', 'key': 'safe_value'}, 'field_application_how': {'id': '177d56e8-2df1-4ead-8e3d-4916610fbed6', 'type': 'value', 'key': 'safe_value'}, 'field_application_insight': {'id': '05ff1755-947b-4e41-8f71-aae99977c572', 'type': 'value', 'key': 'safe_value'}, 'field_application_duration2': {'id': 'duration', 'type': 'value'}, 'field_application_host_text': {'id': '0afaf4e1-4556-4e79-aa3d-4990e33620da', 'type': 'value', 'key': 'safe_value'}, 'field_application_host2_text': {'id': 'a543b34f-ae6a-4b17-8ac3-ececc14573a0', 'type': 'value', 'key': 'safe_value'}, 'field_application_questions': {'id': '57cc52e2-b3ff-4e9f-a5fe-42e7735e16c2', 'type': 'merge_value', 'key': 'safe_value'}, 'field_application_status': {'id': 'ff4d12ff-7b88-4e87-bb5b-81543aef0e25', 'type': 'category', 'key': 'tid'}, 'field_application_objectives': {'id': '30c41288-a762-4003-acce-8c12e7343d90', 'type': 'category', 'key': 'tid'}, 'field_application_beneficiaries': {'id': '56833441-542b-4a06-8ad2-8e7e8fd1a334', 'type': 'category', 'key': 'tid'}, 'field_application_focus': {'id': '6b404851-ce2b-494f-b9f7-62858a937469', 'type': 'category', 'key': 'tid'}, 'field_application_problems': {'id': '590e4b77-c4f4-4bd0-b5be-2ad2851da4f5', 'type': 'category', 'key': 'tid'}, 'field_term_region': {'id': '81c01278-8ba4-4d84-a1da-e05a07aad874', 'type': 'category', 'key': 'tid'}, 'field_concept_upload': {'id': '25740b9d-0f8f-4ce1-88fa-c6ee831c6aef', 'type': 'file'}, 'field_application_otf_mission': {'id': '5178e15f-d442-4d36-824d-a4292ef77062', 'type': 'boolean'}, 'field_application_otf_tos': {'id': 'bd91e220-4cdb-4392-8054-7b7dfe667d46', 'type': 'boolean'}, 'field_application_otf_represent': {'id': '8d000129-ca8b-48cf-8dc2-4651bcbe46e8', 'type': 'boolean'}, 'field_application_otf_license': {'id': '92f0801e-b9dc-4edc-9716-3f1709ae1c9b', 'type': 'boolean'}, 'field_application_otf_complete': {'id': '3a3f2da3-4e32-4b86-9060-29c606927114', 'type': 'boolean'}, 'field_application_otf_deadline': {'id': '19395179-ed9f-4556-9b6b-ab5caef4f610', 'type': 'boolean'}, 'field_application_otf_list': {'id': '1345a8eb-4dcc-4170-a5ac-edda42d4dafc', 'type': 'boolean'}, 'field_application_otf_newsletter': {'id': '4ca22ebb-daba-4fb6-a4a6-b130dc6311a8', 'type': 'boolean'}}

REQUEST_QUESTION_MAP class-attribute instance-attribute

REQUEST_QUESTION_MAP = {'3618': {0: 'What will be the outcome(s) of your research?', 1: 'How will the results of your research be accessible to a non-technical audience?', 2: 'How will your work build on the existing research in your area of focus?', 3: 'Why is the organization chosen well suited to host your project?', 4: 'Please include a resume or CV (Feel free to attach a file at the bottom of the application)'}, '3667': {0: 'Does your fellowship project address an urgent and time-bound digital emergency? If so, how? ', 1: 'What steps will you take post-emergency to prevent a similar problem from occurring again in the future? '}, '3681': {0: 'How does your fellowship project idea address the digital security threats the host organization(s) face?', 1: 'What are the anticipated results from your fellowship project?', 2: 'How will the results of your fellowship project be accessible to internet freedom technology developer community?', 3: 'Why is the organization(s) chosen well suited for your idea under the DIFP remit?', 4: 'Please include a resume or CV (Feel free to attach a file at the bottom of the application)'}, '3861': {0: 'Where are you located or would you like to be located during this fellowship?', 1: 'When would you ideally start and why?', 2: 'Please list any links to your portfolio or prior work'}}

add_arguments

add_arguments(parser)

Source code in hypha/apply/funds/management/commands/migration_base.py

def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle

handle(*args, **options)

Source code in hypha/apply/funds/management/commands/migration_base.py

@transaction.atomic
def handle(self, *args, **options):
    # Prepare the list of categories.
    for item in CATEGORIES:
        category, _ = Category.objects.get_or_create(name=item["category"])
        option, _ = Option.objects.get_or_create(
            value=item["name"], category=category
        )
        self.terms[item["tid"]] = option

    with options["source"] as json_data:
        self.data = json.load(json_data)

        counter = 0
        for id in self.data:
            self.process(id)
            counter += 1

        self.stdout.write(f"Imported {counter} submissions.")

process

process(id)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process(self, id):
    node = self.data[id]

    try:
        submission = ApplicationSubmission.objects.get(drupal_id=node["nid"])
    except ApplicationSubmission.DoesNotExist:
        submission = ApplicationSubmission(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in submission._meta.local_fields:
        if field.name == "submit_time":
            field.auto_now_add = False

    # TODO timezone?
    submission.submit_time = datetime.fromtimestamp(
        int(node["created"]), timezone.utc
    )
    submission.user = self.get_user(node["uid"])

    if self.CONTENT_TYPE == "fund":
        FUND = FundType.objects.get(title=self.FUND_NAME)
        submission.page = FUND
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        submission.round = ROUND
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[1]
        submission.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        submission.page = LAB
        FORM = LabBaseForm.objects.get(lab=LAB)
        submission.form_fields = FORM.form.form_fields

    submission.status = self.get_workflow_state(node)

    if "proposal_nid" in node:
        try:
            submission.next = ApplicationSubmission.objects.get(
                drupal_id=node["proposal_nid"]
            )
        except ApplicationSubmission.DoesNotExist:
            self.stdout.write(
                "No related proposal found, please import proposals before applications."
            )
            pass

    form_data = {
        "skip_account_creation_notification": True,
    }

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "value" not in form_data or not form_data["value"]:
        form_data["value"] = 0

    if "duration" not in form_data or not form_data["duration"]:
        form_data["duration"] = "1"

    if "email" not in form_data or not form_data["email"]:
        if hasattr(submission.user, "email"):
            form_data["email"] = submission.user.email
        else:
            form_data["email"] = f"user+{node['uid']}@example.com"

    submission.form_data = form_data

    try:
        submission.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user

get_user(uid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[
                node["field_application_request"]["target_id"]
            ]
            values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "address" and "map" in mapping:
        try:
            value_map = mapping["map"]
            value = {}
            for item in value_map:
                value[value_map[item]] = source_value[item]
            value = json.dumps(value)
        except TypeError:
            value = {}
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False
    elif mapping_type == "category":
        if not source_value:
            value = []
        else:
            if isinstance(source_value, dict):
                option = self.get_referenced_term(source_value[key])
                value = [option] if option else []
            else:
                value = []
                for item in source_value:
                    option = self.get_referenced_term(item[key])
                    if option:
                        value.append(option)
    elif mapping_type == "file":
        value = self.process_file(source_value)

    return value

get_referenced_term

get_referenced_term(tid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_term(self, tid):
    try:
        term = self.terms[tid]
        return term.id
    except KeyError:
        return None

get_referenced_node

get_referenced_node(nid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_node(self, nid):
    pass

get_workflow_state

get_workflow_state(node)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_workflow_state(self, node):
    """
    workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}}
    """
    states_request = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "internal_review",
        "ready_for_reply": "post_review_discussion",
        "contract_review": "post_review_discussion",
        "in_contract": "accepted",
        "invited_for_proposal": "accepted",
        "dropped_concept_note": "rejected",
        "dropped": "rejected",
        "dropped_without_review": "rejected",
    }

    states_concept = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "concept_internal_review",
        "ready_for_reply": "concept_review_discussion",
        "contract_review": "concept_review_discussion",
        "in_contract": "invited_to_proposal",
        "invited_for_proposal": "invited_to_proposal",
        "dropped_concept_note": "concept_rejected",
        "dropped": "concept_rejected",
        "dropped_without_review": "concept_rejected",
    }

    states_proposal = {
        "draft": "draft_proposal",
        "published": "proposal_discussion",
        "in_discussion": "proposal_discussion",
        "council_review": "external_review",
        "ready_for_reply": "proposal_more_info",
        "contract_review": "post_external_review_discussion",
        "in_contract": "proposal_accepted",
        "invited_for_proposal": "proposal_accepted",
        "dropped_concept_note": "proposal_rejected",
        "dropped": "proposal_rejected",
        "dropped_without_review": "proposal_rejected",
    }

    if self.APPLICATION_TYPE == "request":
        workflow_state = states_request.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "concept":
        workflow_state = states_concept.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "proposal":
        workflow_state = states_proposal.get(
            node["workbench_moderation"]["current"]["state"], "draft_proposal"
        )
    else:
        workflow_state = None

    return workflow_state

nl2br

nl2br(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")

process_file

process_file(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process_file(self, value):
    if isinstance(value, dict):
        value = [value]

    files = []

    for file_data in value:
        parts = urlsplit(file_data["uri"])
        file_path = os.path.join(
            "files", "private", parts.netloc, *parts.path.split("/")
        )
        saved_file = migration_storage.open(file_path)
        saved_file.name = file_data["filename"]
        files.append(saved_file)

    return files