Skip to content

Migrate rr applications

hypha.apply.funds.management.commands.migrate_rr_applications

Command

Bases: MigrateCommand

help class-attribute instance-attribute 

help = 'Application migration script. Requires a source JSON file.'

data class-attribute instance-attribute 

data = []

terms class-attribute instance-attribute 

terms = {}

patched_status_field class-attribute instance-attribute 

patched_status_field = FSMField(default=INITIAL_STATE, protected=False)

CONTENT_TYPE class-attribute instance-attribute 

CONTENT_TYPE = 'fund'

FUND_NAME class-attribute instance-attribute 

FUND_NAME = 'Rapid Response (archive fund)'

ROUND_NAME class-attribute instance-attribute 

ROUND_NAME = 'Rapid Response (archive round)'

APPLICATION_TYPE class-attribute instance-attribute 

APPLICATION_TYPE = 'request'

STREAMFIELD_MAP class-attribute instance-attribute 

STREAMFIELD_MAP = {'title': {'id': 'title', 'type': 'direct'}, 'field_application_name': {'id': 'full_name', 'type': 'value', 'key': 'safe_value'}, 'field_application_mail': {'id': 'email', 'type': 'value', 'key': 'email'}, 'field_application_preapplied': {'id': '49a0f5f4-e8e9-4dfc-8575-205ee9675032', 'type': 'map', 'map': {'0': 'No', '1': 'Yes'}}, 'field_application_apply_for': {'id': 'c1277029-1718-40e3-8bf5-d80ece7fd343', 'type': 'map', 'map': {'direct': 'Direct funding', 'receive': 'Requesting to receive services', 'provide': 'Requesting to provide services'}}, 'field_application_amount': {'id': 'value', 'type': 'value'}, 'field_application_amount_text': {'id': 'value', 'type': 'value'}, 'field_application_service': {'id': 'ebdf9a22-58c7-4bd6-a58d-e71363357470', 'type': 'map', 'map': {'audit': 'Audit of presumably compromised websites', 'ddos': 'DDoS response and mitigation', 'hosting': 'Secure web hosting', 'hostingevents': 'Secure hosting for monitoring and resiliency of websites during special events (elections, campaigns etc.)', 'vpn': 'VPN connections', 'isp': 'Safe internet connections', 'analysis': 'Forensic analysis of digital attacks', 'recovery': 'Recovery of compromised websites', 'malware': 'Malware analysis', 'equipment': 'Equipment replacements (unavailable)', 'legalhelp': 'Finding legal representation (unavailable)', 'legalfees': 'Payment of legal fees (unavailable)'}}, 'field_application_service_other': {'id': 'c8c329c7-78e4-4cbf-a3b1-77a1324e92ff', 'type': 'value', 'key': 'safe_value'}, 'field_application_duration3': {'id': 'duration', 'type': 'value'}, 'field_application_who': {'id': '1ec16cdc-7a68-40be-b17b-9a218def4260', 'type': 'value', 'key': 'safe_value'}, 'field_application_how': {'id': '4fa2ac11-d1cd-4d23-8082-93a14c8f99c8', 'type': 'value', 'key': 'safe_value'}, 'field_application_sustainability': {'id': '3cde39ae-b687-4c4f-b58b-849396c2fdb8', 'type': 'value', 'key': 'safe_value'}, 'field_application_dates': {'id': '0b2a4653-b390-44a6-b92e-fae4647e7ec4', 'type': 'value', 'key': 'safe_value'}, 'field_application_why': {'id': '6d75e412-cf53-4833-9f1d-3e0126512fb9', 'type': 'value', 'key': 'safe_value'}, 'field_application_why_rapiid': {'id': '1b181d1e-ef91-41af-b9c1-d096a991314b', 'type': 'value', 'key': 'safe_value'}, 'field_application_focus': {'id': 'efd91eaf-378f-4aab-96cb-c5601155cbee', 'type': 'category', 'key': 'tid'}, 'field_application_objectives': {'id': '4be0c7bd-231d-4d9f-bd47-8589fc005f54', 'type': 'category', 'key': 'tid'}, 'field_application_beneficiaries': {'id': '6e0293ee-218e-4c3b-b82d-5bf91fdb21c9', 'type': 'category', 'key': 'tid'}, 'field_term_region': {'id': '6ff029c6-c6d1-4c37-a49a-46181b1cd33d', 'type': 'category', 'key': 'tid'}, 'field_application_problems': {'id': '7fb1001e-d458-414f-a5bb-006db6f89baf', 'type': 'category', 'key': 'tid'}, 'field_application_budget': {'id': '45d7d38a-9c9d-4c43-98df-bb95d4a1dd77', 'type': 'value', 'key': 'safe_value'}, 'field_application_legal_name': {'id': '632065c5-860f-4751-9b31-52914d7c6448', 'type': 'value', 'key': 'safe_value'}, 'field_application_contact': {'id': '13bb0d64-65f3-4340-8e7e-e5da80d706d5', 'type': 'value', 'key': 'safe_value'}, 'field_application_phone': {'id': '2cb9fe4b-df45-4181-80e5-14382f853081', 'type': 'value', 'key': 'safe_value'}, 'field_application_address': {'id': 'bd29eb88-9754-4305-9b2d-406a875ec56a', 'type': 'address', 'map': {'administrative_area': 'administrativearea', 'country': 'country', 'locality': 'localityname', 'postal_code': 'postalcode', 'thoroughfare': 'thoroughfare', 'premise': 'premise'}}, 'field_application_questions': {'id': '1889de86-0a0d-4abf-9916-4db87a499d35', 'type': 'merge_value', 'key': 'safe_value'}, 'field_application_otf_mission': {'id': 'e695f0d7-4c74-4cc6-853f-bd62ecd19d3d', 'type': 'boolean'}, 'field_application_otf_tos': {'id': 'f40d1acc-d802-4cc6-b0e9-fff78dc54223', 'type': 'boolean'}, 'field_application_otf_represent': {'id': '0b3c0827-38e2-439b-bca5-735835af1019', 'type': 'boolean'}, 'field_application_otf_license': {'id': 'bc9c960e-a6f4-4bc2-b626-efb5bc5552c6', 'type': 'boolean'}, 'field_application_otf_complete': {'id': '5812b66d-630e-4ca2-8bea-819084278f55', 'type': 'boolean'}, 'field_application_otf_deadline': {'id': '97d3746c-cf0f-449a-b3a3-7a9cdd45cc6d', 'type': 'boolean'}, 'field_application_otf_list': {'id': 'fc3d2a87-1151-418b-b1cd-9289f00bde35', 'type': 'boolean'}, 'field_application_otf_newsletter': {'id': '83ecc69a-f47c-495e-bc8f-326e55aed67a', 'type': 'boolean'}, 'field_concept_upload': {'id': '607daeba-1f33-4ad0-b135-eda743ba8e3a', 'type': 'file'}}

REQUEST_QUESTION_MAP class-attribute instance-attribute 

REQUEST_QUESTION_MAP = {'3618': {'0': 'What will be the outcome(s) of your research?', '1': 'How will the results of your research be accessible to a non-technical audience?', '2': 'How will your work build on the existing research in your area of focus?', '3': 'Why is the organization chosen well suited to host your project?', '4': 'Please include a resume or CV (Feel free to attach a file at the bottom of the application)'}, '3667': {'0': 'Does your fellowship project address an urgent and time-bound digital emergency? If so, how? ', '1': 'What steps will you take post-emergency to prevent a similar problem from occurring again in the future? '}, '3681': {'0': 'How does your fellowship project idea address the digital security threats the host organization(s) face?', '1': 'What are the anticipated results from your fellowship project?', '2': 'How will the results of your fellowship project be accessible to internet freedom technology developer community?', '3': 'Why is the organization(s) chosen well suited for your idea under the DIFP remit?', '4': 'Please include a resume or CV (Feel free to attach a file at the bottom of the application)'}, '3861': {'0': 'Where are you located or would you like to be located during this fellowship?', '1': 'When would you ideally start and why?', '2': 'Please list any links to your portfolio or prior work'}}

add_arguments 

add_arguments(parser)
Source code in hypha/apply/funds/management/commands/migration_base.py 
53
54
55
56
def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle 

handle(*args, **options)
Source code in hypha/apply/funds/management/commands/migration_base.py 
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
@transaction.atomic
def handle(self, *args, **options):
    # Prepare the list of categories.
    for item in CATEGORIES:
        category, _ = Category.objects.get_or_create(name=item["category"])
        option, _ = Option.objects.get_or_create(
            value=item["name"], category=category
        )
        self.terms[item["tid"]] = option

    with options["source"] as json_data:
        self.data = json.load(json_data)

        counter = 0
        for id in self.data:
            self.process(id)
            counter += 1

        self.stdout.write(f"Imported {counter} submissions.")

process 

process(id)
Source code in hypha/apply/funds/management/commands/migration_base.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def process(self, id):
    node = self.data[id]

    try:
        submission = ApplicationSubmission.objects.get(drupal_id=node["nid"])
    except ApplicationSubmission.DoesNotExist:
        submission = ApplicationSubmission(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in submission._meta.local_fields:
        if field.name == "submit_time":
            field.auto_now_add = False

    # TODO timezone?
    submission.submit_time = datetime.fromtimestamp(
        int(node["created"]), timezone.utc
    )
    submission.user = self.get_user(node["uid"])

    if self.CONTENT_TYPE == "fund":
        FUND = FundType.objects.get(title=self.FUND_NAME)
        submission.page = FUND
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        submission.round = ROUND
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseForm.objects.filter(round=ROUND)[1]
        submission.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        submission.page = LAB
        FORM = LabBaseForm.objects.get(lab=LAB)
        submission.form_fields = FORM.form.form_fields

    submission.status = self.get_workflow_state(node)

    if "proposal_nid" in node:
        try:
            submission.next = ApplicationSubmission.objects.get(
                drupal_id=node["proposal_nid"]
            )
        except ApplicationSubmission.DoesNotExist:
            self.stdout.write(
                "No related proposal found, please import proposals before applications."
            )
            pass

    form_data = {
        "skip_account_creation_notification": True,
    }

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "value" not in form_data or not form_data["value"]:
        form_data["value"] = 0

    if "duration" not in form_data or not form_data["duration"]:
        form_data["duration"] = "1"

    if "email" not in form_data or not form_data["email"]:
        if hasattr(submission.user, "email"):
            form_data["email"] = submission.user.email
        else:
            form_data["email"] = f"user+{node['uid']}@example.com"

    submission.form_data = form_data

    try:
        submission.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user 

get_user(uid)
Source code in hypha/apply/funds/management/commands/migration_base.py 
162
163
164
165
166
167
def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value 

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/funds/management/commands/migration_base.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[
                node["field_application_request"]["target_id"]
            ]
            values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "address" and "map" in mapping:
        try:
            value_map = mapping["map"]
            value = {}
            for item in value_map:
                value[value_map[item]] = source_value[item]
            value = json.dumps(value)
        except TypeError:
            value = {}
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False
    elif mapping_type == "category":
        if not source_value:
            value = []
        else:
            if isinstance(source_value, dict):
                option = self.get_referenced_term(source_value[key])
                value = [option] if option else []
            else:
                value = []
                for item in source_value:
                    option = self.get_referenced_term(item[key])
                    if option:
                        value.append(option)
    elif mapping_type == "file":
        value = self.process_file(source_value)

    return value

get_referenced_term 

get_referenced_term(tid)
Source code in hypha/apply/funds/management/commands/migration_base.py 
233
234
235
236
237
238
def get_referenced_term(self, tid):
    try:
        term = self.terms[tid]
        return term.id
    except KeyError:
        return None

get_referenced_node 

get_referenced_node(nid)
Source code in hypha/apply/funds/management/commands/migration_base.py 
240
241
def get_referenced_node(self, nid):
    pass

get_workflow_state 

get_workflow_state(node)
Source code in hypha/apply/funds/management/commands/migration_base.py 
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
def get_workflow_state(self, node):
    """
    workbench_moderation: {'current': {'state': STATE, 'timestamp': TS}}
    """
    states_request = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "internal_review",
        "ready_for_reply": "post_review_discussion",
        "contract_review": "post_review_discussion",
        "in_contract": "accepted",
        "invited_for_proposal": "accepted",
        "dropped_concept_note": "rejected",
        "dropped": "rejected",
        "dropped_without_review": "rejected",
    }

    states_concept = {
        "draft": "in_discussion",
        "published": "in_discussion",
        "in_discussion": "in_discussion",
        "council_review": "concept_internal_review",
        "ready_for_reply": "concept_review_discussion",
        "contract_review": "concept_review_discussion",
        "in_contract": "invited_to_proposal",
        "invited_for_proposal": "invited_to_proposal",
        "dropped_concept_note": "concept_rejected",
        "dropped": "concept_rejected",
        "dropped_without_review": "concept_rejected",
    }

    states_proposal = {
        "draft": "draft_proposal",
        "published": "proposal_discussion",
        "in_discussion": "proposal_discussion",
        "council_review": "external_review",
        "ready_for_reply": "proposal_more_info",
        "contract_review": "post_external_review_discussion",
        "in_contract": "proposal_accepted",
        "invited_for_proposal": "proposal_accepted",
        "dropped_concept_note": "proposal_rejected",
        "dropped": "proposal_rejected",
        "dropped_without_review": "proposal_rejected",
    }

    if self.APPLICATION_TYPE == "request":
        workflow_state = states_request.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "concept":
        workflow_state = states_concept.get(
            node["workbench_moderation"]["current"]["state"], "in_discussion"
        )
    elif self.APPLICATION_TYPE == "proposal":
        workflow_state = states_proposal.get(
            node["workbench_moderation"]["current"]["state"], "draft_proposal"
        )
    else:
        workflow_state = None

    return workflow_state

nl2br 

nl2br(value)
Source code in hypha/apply/funds/management/commands/migration_base.py 
306
307
def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")

process_file 

process_file(value)
Source code in hypha/apply/funds/management/commands/migration_base.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
def process_file(self, value):
    if isinstance(value, dict):
        value = [value]

    files = []

    for file_data in value:
        parts = urlsplit(file_data["uri"])
        file_path = os.path.join(
            "files", "private", parts.netloc, *parts.path.split("/")
        )
        saved_file = migration_storage.open(file_path)
        saved_file.name = file_data["filename"]
        files.append(saved_file)

    return files