Migrate concept notes 502

hypha.apply.funds.management.commands.migrate_concept_notes_502

Command

Bases: MigrateCommand

help class-attribute instance-attribute

help = 'Application migration script. Requires a source JSON file.'

data class-attribute instance-attribute

data = []

terms class-attribute instance-attribute

terms = {}

patched_status_field class-attribute instance-attribute

patched_status_field = FSMField(default=INITIAL_STATE, protected=False)

CONTENT_TYPE class-attribute instance-attribute

CONTENT_TYPE = 'fund'

FUND_NAME class-attribute instance-attribute

FUND_NAME = 'Internet Freedom Fund'

ROUND_NAME class-attribute instance-attribute

ROUND_NAME = 'IFFund-2018-09'

APPLICATION_TYPE class-attribute instance-attribute

APPLICATION_TYPE = 'concept'

STREAMFIELD_MAP class-attribute instance-attribute

STREAMFIELD_MAP = {'title': {'id': 'title', 'type': 'direct'}, 'field_concept_name': {'id': 'full_name', 'type': 'value', 'key': 'safe_value'}, 'field_concept_mail': {'id': 'email', 'type': 'value', 'key': 'email'}, 'field_concept_preapplied': {'id': '0305a465-8763-4c1f-9197-4ca4227d452a', 'type': 'map', 'map': {'0': 'No', '1': 'Yes'}}, 'field_concept_preapplied_how': {'id': 'c24691be-9861-4dbc-8be4-03b6e68c1973', 'type': 'value', 'key': 'safe_value'}, 'field_concept_description': {'id': 'c21c58c3-cfbe-4409-b2f2-8f56398f1731', 'type': 'value', 'key': 'safe_value'}, 'field_concept_outcome': {'id': '27289c14-6926-4f61-bea2-8031a653f71c', 'type': 'value', 'key': 'safe_value'}, 'field_type': {'id': '404e2310-000b-4ccb-b772-3680946ff07d', 'type': 'category', 'key': 'tid'}, 'field_proposal_status': {'id': '145c364f-e0bb-4652-94e8-fe08c831da2b', 'type': 'category', 'key': 'tid'}, 'field_technology_attribute': {'id': 'b4da2310-9654-4aa7-a04a-06335967ddc5', 'type': 'category', 'key': 'tid'}, 'field_concept_how': {'id': '418b8099-4525-437f-a55c-9b35745d0384', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_focus': {'id': '390702bd-e4e1-4dc2-8c43-d51bf018b427', 'type': 'category', 'key': 'tid'}, 'field_concept_time': {'id': 'duration', 'type': 'value'}, 'field_concept_amount': {'id': 'value', 'type': 'value'}, 'field_concept_how_long': {'id': 'dfb9c307-9328-4a99-9efc-321d474b2ba7', 'type': 'value', 'key': 'safe_value'}, 'field_concept_who': {'id': '11f94a22-0571-4491-a93e-87c050e02a4a', 'type': 'value', 'key': 'safe_value'}, 'field_concept_community': {'id': 'c272969b-d89f-4b6e-859f-4606a15b3f28', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_beneficiaries': {'id': '72002c3e-aaee-47da-9377-8bb493f14c21', 'type': 'category', 'key': 'tid'}, 'field_term_region': {'id': '369b248e-f669-4aeb-b771-7cba0eadb921', 'type': 'category', 'key': 'tid'}, 'field_term_country': {'id': '57bceb33-ebda-4708-9080-fd1a5923e008', 'type': 'category', 'key': 'tid'}, 'field_concept_why': {'id': 'c14ee077-c0eb-48b1-9825-fbba9b91ede5', 'type': 'value', 'key': 'safe_value'}, 'field_proposal_theme': {'id': 'a83a1884-f711-4196-8d15-ae2110466acb', 'type': 'category', 'key': 'tid'}, 'field_proposal_similar_efforts': {'id': '9ca35708-d611-4cd0-8d4a-3cc08349f45b', 'type': 'value', 'key': 'safe_value'}, 'field_concept_contact': {'id': 'db7b1642-c03d-4af4-82c9-db67bf9713b0', 'type': 'value', 'key': 'safe_value'}, 'field_concept_upload': {'id': '8a91231c-5c3d-46fe-9de6-8d5c86817626', 'type': 'file'}, 'field_application_otf_mission': {'id': '4c661a64-2614-4169-b4d2-1fd39e2e831b', 'type': 'boolean'}, 'field_application_otf_tos': {'id': '1bc4e113-1414-46ff-bba7-2dc02b2126df', 'type': 'boolean'}, 'field_application_otf_represent': {'id': '42dd68a1-b699-4678-bea6-13e0f842e821', 'type': 'boolean'}, 'field_application_otf_license': {'id': '72916731-ec97-4688-95f1-d3bf140b03c2', 'type': 'boolean'}, 'field_application_otf_complete': {'id': '6856d26d-b169-4fdf-b598-63c3dd9278a2', 'type': 'boolean'}, 'field_application_otf_deadline': {'id': '33838399-f292-4b63-83f0-e02d344f99d4', 'type': 'boolean'}, 'field_application_otf_list': {'id': 'fc571e12-d4a2-4d53-ab34-2c57321dc6ac', 'type': 'boolean'}, 'field_application_otf_newsletter': {'id': 'cd0d8a4b-e71a-4dff-964a-f547bd655e7d', 'type': 'boolean'}}

add_arguments

add_arguments(parser)

Source code in hypha/apply/funds/management/commands/migration_base.py

def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle

handle(*args, **options)

Source code in hypha/apply/funds/management/commands/migration_base.py

@transaction.atomic
def handle(self, *args, **options):
    # Prepare the list of categories.
    for item in CATEGORIES:
        category, _ = Category.objects.get_or_create(name=item["category"])
        option, _ = Option.objects.get_or_create(
            value=item["name"], category=category
        )
        self.terms[item["tid"]] = option

    with options["source"] as json_data:
        self.data = json.load(json_data)

        counter = 0
        for id in self.data:
            self.process(id)
            counter += 1

        self.stdout.write(f"Imported {counter} submissions.")

process

process(id)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process(self, id):
    node = self.data[id]

    try:
        submission = ApplicationSubmission.objects.get(drupal_id=node["nid"])
    except ApplicationSubmission.DoesNotExist:
        submission = ApplicationSubmission(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in submission._meta.local_fields:
        if field.name == "submit_time":
            field.auto_now_add = False

    # TODO timezone?
    submission.submit_time = datetime.fromtimestamp(
        int(node["created"]), timezone.utc
    )
    submission.user = self.get_user(node["uid"])

    if self.CONTENT_TYPE == "fund":
        FUND = FundType.objects.get(title=self.FUND_NAME)
        submission.page = FUND
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        submission.round = ROUND
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[1]
        submission.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        submission.page = LAB
        FORM = LabBaseForm.objects.get(lab=LAB)
        submission.form_fields = FORM.form.form_fields

    submission.status = self.get_workflow_state(node)

    if "proposal_nid" in node:
        try:
            submission.next = ApplicationSubmission.objects.get(
                drupal_id=node["proposal_nid"]
            )
        except ApplicationSubmission.DoesNotExist:
            self.stdout.write(
                "No related proposal found, please import proposals before applications."
            )
            pass

    form_data = {
        "skip_account_creation_notification": True,
    }

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "value" not in form_data or not form_data["value"]:
        form_data["value"] = 0

    if "duration" not in form_data or not form_data["duration"]:
        form_data["duration"] = "1"

    if "email" not in form_data or not form_data["email"]:
        if hasattr(submission.user, "email"):
            form_data["email"] = submission.user.email
        else:
            form_data["email"] = f"user+{node['uid']}@example.com"

    submission.form_data = form_data

    try:
        submission.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user

get_user(uid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[
                node["field_application_request"]["target_id"]
            ]
            values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "address" and "map" in mapping:
        try:
            value_map = mapping["map"]
            value = {}
            for item in value_map:
                value[value_map[item]] = source_value[item]
            value = json.dumps(value)
        except TypeError:
            value = {}
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False
    elif mapping_type == "category":
        if not source_value:
            value = []
        else:
            if isinstance(source_value, dict):
                option = self.get_referenced_term(source_value[key])
                value = [option] if option else []
            else:
                value = []
                for item in source_value:
                    option = self.get_referenced_term(item[key])
                    if option:
                        value.append(option)
    elif mapping_type == "file":
        value = self.process_file(source_value)

    return value

get_referenced_term

get_referenced_term(tid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_term(self, tid):
    try:
        term = self.terms[tid]
        return term.id
    except KeyError:
        return None

get_referenced_node

get_referenced_node(nid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_node(self, nid):
    pass

get_workflow_state

get_workflow_state(node)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_workflow_state(self, node):
    """
    workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}}
    """
    states_request = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "internal_review",
        "ready_for_reply": "post_review_discussion",
        "contract_review": "post_review_discussion",
        "in_contract": "accepted",
        "invited_for_proposal": "accepted",
        "dropped_concept_note": "rejected",
        "dropped": "rejected",
        "dropped_without_review": "rejected",
    }

    states_concept = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "concept_internal_review",
        "ready_for_reply": "concept_review_discussion",
        "contract_review": "concept_review_discussion",
        "in_contract": "invited_to_proposal",
        "invited_for_proposal": "invited_to_proposal",
        "dropped_concept_note": "concept_rejected",
        "dropped": "concept_rejected",
        "dropped_without_review": "concept_rejected",
    }

    states_proposal = {
        "draft": "draft_proposal",
        "published": "proposal_discussion",
        "in_discussion": "proposal_discussion",
        "council_review": "external_review",
        "ready_for_reply": "proposal_more_info",
        "contract_review": "post_external_review_discussion",
        "in_contract": "proposal_accepted",
        "invited_for_proposal": "proposal_accepted",
        "dropped_concept_note": "proposal_rejected",
        "dropped": "proposal_rejected",
        "dropped_without_review": "proposal_rejected",
    }

    if self.APPLICATION_TYPE == "request":
        workflow_state = states_request.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "concept":
        workflow_state = states_concept.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "proposal":
        workflow_state = states_proposal.get(
            node["workbench_moderation"]["current"]["state"], "draft_proposal"
        )
    else:
        workflow_state = None

    return workflow_state

nl2br

nl2br(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")

process_file

process_file(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process_file(self, value):
    if isinstance(value, dict):
        value = [value]

    files = []

    for file_data in value:
        parts = urlsplit(file_data["uri"])
        file_path = os.path.join(
            "files", "private", parts.netloc, *parts.path.split("/")
        )
        saved_file = migration_storage.open(file_path)
        saved_file.name = file_data["filename"]
        files.append(saved_file)

    return files