Migrate community lab applications

hypha.apply.funds.management.commands.migrate_community_lab_applications

Command

Bases: MigrateCommand

help class-attribute instance-attribute

help = 'Application migration script. Requires a source JSON file.'

data class-attribute instance-attribute

data = []

terms class-attribute instance-attribute

terms = {}

patched_status_field class-attribute instance-attribute

patched_status_field = FSMField(default=INITIAL_STATE, protected=False)

LAB_NAME class-attribute instance-attribute

LAB_NAME = 'Community lab (archive fund)'

APPLICATION_TYPE class-attribute instance-attribute

APPLICATION_TYPE = 'request'

CONTENT_TYPE class-attribute instance-attribute

CONTENT_TYPE = 'lab'

STREAMFIELD_MAP class-attribute instance-attribute

STREAMFIELD_MAP = {'title': {'id': 'title', 'type': 'direct'}, 'field_application_name': {'id': 'full_name', 'type': 'value', 'key': 'safe_value'}, 'field_application_mail': {'id': 'email', 'type': 'value', 'key': 'email'}, 'field_application_event_date': {'id': 'd3364b5b-976b-4cd6-85d5-4ccc0bb9b560', 'type': 'value', 'key': 'value'}, 'field_application_amount': {'id': 'value', 'type': 'value'}, 'field_application_amount_text': {'id': 'value', 'type': 'value'}, 'field_application_describe': {'id': 'aeea7bd9-01b5-406e-bb30-1303ba0f550a', 'type': 'value', 'key': 'safe_value'}, 'field_application_who': {'id': 'a5117396-0286-4937-91a3-be33a5944ac6', 'type': 'value', 'key': 'safe_value'}, 'field_application_objective_text': {'id': 'd90d2cd4-831e-46f5-8f2e-82ac94913784', 'type': 'value', 'key': 'safe_value'}, 'field_application_strategy': {'id': 'fc5ddac7-c7b5-4d07-9bc3-16902e9a7afb', 'type': 'value', 'key': 'safe_value'}, 'field_application_how': {'id': 'ea6ceb76-882e-4739-94da-7b055a112d1c', 'type': 'value', 'key': 'safe_value'}, 'field_application_collaboration': {'id': 'afa7e16b-d31c-4016-af88-7939acc7b6e1', 'type': 'value', 'key': 'safe_value'}, 'field_application_diverse': {'id': '55e23c45-72a7-4ab9-9940-764d00724de8', 'type': 'value', 'key': 'safe_value'}, 'field_application_outreach': {'id': '737bd894-78b1-41ef-969c-2a57c843cd5b', 'type': 'value', 'key': 'safe_value'}, 'field_application_needs': {'id': '56c84f40-f59e-4e82-80a1-2cd971717e9c', 'type': 'value', 'key': 'safe_value'}, 'field_application_budget': {'id': '6ca29ae0-780a-467a-a3e9-34195bcb0d79', 'type': 'value', 'key': 'safe_value'}, 'field_application_cod': {'id': '9db8b2ca-62b6-44c4-9d3d-70cb4a28e65f', 'type': 'boolean'}, 'field_concept_upload': {'id': 'b3af7aac-3439-45fa-9573-518f82f5cd6c', 'type': 'file'}, 'field_application_otf_mission': {'id': '1248f597-2f18-4b16-8f96-63912e5197c5', 'type': 'boolean'}, 'field_application_otf_tos': {'id': '5a676552-e189-417e-9901-05bfc973cfb5', 'type': 'boolean'}, 'field_application_otf_represent': {'id': 'c4f54c3d-6b2a-4b32-b651-9121430aa06f', 'type': 'boolean'}, 'field_application_otf_license': {'id': 'beb4d454-466d-43d5-823e-80dbccacbbb3', 'type': 'boolean'}, 'field_application_otf_complete': {'id': 'fd6b034d-7cec-49fe-b4da-991c382283ef', 'type': 'boolean'}, 'field_application_otf_deadline': {'id': 'd930a864-263d-4d0d-8ff1-553b13efda41', 'type': 'boolean'}, 'field_application_otf_list': {'id': 'f75fd335-be7d-460d-b96a-53d7aa9a826f', 'type': 'boolean'}, 'field_application_otf_newsletter': {'id': 'a1f03bca-9267-49cf-9880-444d6806065c', 'type': 'boolean'}}

add_arguments

add_arguments(parser)

Source code in hypha/apply/funds/management/commands/migration_base.py

def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle

handle(*args, **options)

Source code in hypha/apply/funds/management/commands/migration_base.py

@transaction.atomic
def handle(self, *args, **options):
    # Prepare the list of categories.
    for item in CATEGORIES:
        category, _ = Category.objects.get_or_create(name=item["category"])
        option, _ = Option.objects.get_or_create(
            value=item["name"], category=category
        )
        self.terms[item["tid"]] = option

    with options["source"] as json_data:
        self.data = json.load(json_data)

        counter = 0
        for id in self.data:
            self.process(id)
            counter += 1

        self.stdout.write(f"Imported {counter} submissions.")

process

process(id)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process(self, id):
    node = self.data[id]

    try:
        submission = ApplicationSubmission.objects.get(drupal_id=node["nid"])
    except ApplicationSubmission.DoesNotExist:
        submission = ApplicationSubmission(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in submission._meta.local_fields:
        if field.name == "submit_time":
            field.auto_now_add = False

    # TODO timezone?
    submission.submit_time = datetime.fromtimestamp(
        int(node["created"]), timezone.utc
    )
    submission.user = self.get_user(node["uid"])

    if self.CONTENT_TYPE == "fund":
        FUND = FundType.objects.get(title=self.FUND_NAME)
        submission.page = FUND
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        submission.round = ROUND
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[1]
        submission.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        submission.page = LAB
        FORM = LabBaseForm.objects.get(lab=LAB)
        submission.form_fields = FORM.form.form_fields

    submission.status = self.get_workflow_state(node)

    if "proposal_nid" in node:
        try:
            submission.next = ApplicationSubmission.objects.get(
                drupal_id=node["proposal_nid"]
            )
        except ApplicationSubmission.DoesNotExist:
            self.stdout.write(
                "No related proposal found, please import proposals before applications."
            )
            pass

    form_data = {
        "skip_account_creation_notification": True,
    }

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "value" not in form_data or not form_data["value"]:
        form_data["value"] = 0

    if "duration" not in form_data or not form_data["duration"]:
        form_data["duration"] = "1"

    if "email" not in form_data or not form_data["email"]:
        if hasattr(submission.user, "email"):
            form_data["email"] = submission.user.email
        else:
            form_data["email"] = f"user+{node['uid']}@example.com"

    submission.form_data = form_data

    try:
        submission.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user

get_user(uid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[
                node["field_application_request"]["target_id"]
            ]
            values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "address" and "map" in mapping:
        try:
            value_map = mapping["map"]
            value = {}
            for item in value_map:
                value[value_map[item]] = source_value[item]
            value = json.dumps(value)
        except TypeError:
            value = {}
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False
    elif mapping_type == "category":
        if not source_value:
            value = []
        else:
            if isinstance(source_value, dict):
                option = self.get_referenced_term(source_value[key])
                value = [option] if option else []
            else:
                value = []
                for item in source_value:
                    option = self.get_referenced_term(item[key])
                    if option:
                        value.append(option)
    elif mapping_type == "file":
        value = self.process_file(source_value)

    return value

get_referenced_term

get_referenced_term(tid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_term(self, tid):
    try:
        term = self.terms[tid]
        return term.id
    except KeyError:
        return None

get_referenced_node

get_referenced_node(nid)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_referenced_node(self, nid):
    pass

get_workflow_state

get_workflow_state(node)

Source code in hypha/apply/funds/management/commands/migration_base.py

def get_workflow_state(self, node):
    """
    workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}}
    """
    states_request = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "internal_review",
        "ready_for_reply": "post_review_discussion",
        "contract_review": "post_review_discussion",
        "in_contract": "accepted",
        "invited_for_proposal": "accepted",
        "dropped_concept_note": "rejected",
        "dropped": "rejected",
        "dropped_without_review": "rejected",
    }

    states_concept = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "concept_internal_review",
        "ready_for_reply": "concept_review_discussion",
        "contract_review": "concept_review_discussion",
        "in_contract": "invited_to_proposal",
        "invited_for_proposal": "invited_to_proposal",
        "dropped_concept_note": "concept_rejected",
        "dropped": "concept_rejected",
        "dropped_without_review": "concept_rejected",
    }

    states_proposal = {
        "draft": "draft_proposal",
        "published": "proposal_discussion",
        "in_discussion": "proposal_discussion",
        "council_review": "external_review",
        "ready_for_reply": "proposal_more_info",
        "contract_review": "post_external_review_discussion",
        "in_contract": "proposal_accepted",
        "invited_for_proposal": "proposal_accepted",
        "dropped_concept_note": "proposal_rejected",
        "dropped": "proposal_rejected",
        "dropped_without_review": "proposal_rejected",
    }

    if self.APPLICATION_TYPE == "request":
        workflow_state = states_request.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "concept":
        workflow_state = states_concept.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "proposal":
        workflow_state = states_proposal.get(
            node["workbench_moderation"]["current"]["state"], "draft_proposal"
        )
    else:
        workflow_state = None

    return workflow_state

nl2br

nl2br(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")

process_file

process_file(value)

Source code in hypha/apply/funds/management/commands/migration_base.py

def process_file(self, value):
    if isinstance(value, dict):
        value = [value]

    files = []

    for file_data in value:
        parts = urlsplit(file_data["uri"])
        file_path = os.path.join(
            "files", "private", parts.netloc, *parts.path.split("/")
        )
        saved_file = migration_storage.open(file_path)
        saved_file.name = file_data["filename"]
        files.append(saved_file)

    return files