Skip to content

Migrate proposal reviews

hypha.apply.review.management.commands.migrate_proposal_reviews

Command

Bases: MigrateCommand

help class-attribute instance-attribute 

help = 'Review migration script. Requires a source JSON file.'

data class-attribute instance-attribute 

data = []

REVIEWFIELD_MAP class-attribute instance-attribute 

REVIEWFIELD_MAP = {'community_lab_review': {'submission': 'field_review_community_lab', 'recommendation': 'field_clr_recommendation', 'rec_map': {'1': 2, '0': 0}}, 'concept_note_review': {'submission': 'field_review_concept_note', 'recommendation': 'field_pr_recommendation', 'rec_map': {'2': 1, '1': 2, '0': 0}}, 'proposal_review': {'submission': 'field_review_proposal', 'recommendation': 'field_pr_recommendation', 'rec_map': {'2': 1, '1': 2, '0': 0}}, 'fellowship_application_review': {'submission': 'field_review_fellowship_app', 'recommendation': 'field_fr_overall_rate', 'rec_map': {'5': 2, '4': 2, '3': 1, '2': 0, '1': 0}}, 'fellowship_proposal_review': {'submission': 'field_review_fellowship', 'recommendation': 'field_fr_overall_rate', 'rec_map': {'5': 2, '4': 2, '3': 1, '2': 0, '1': 0}}, 'rapid_response_review': {'submission': 'field_review_rapid_response', 'recommendation': 'field_rrr_recommend', 'rec_map': {'3': 2, '2': 1, '1': 0}}}

CONTENT_TYPE class-attribute instance-attribute 

CONTENT_TYPE = 'fund'

FUND_NAME class-attribute instance-attribute 

FUND_NAME = 'Internet Freedom Fund (archive fund)'

ROUND_NAME class-attribute instance-attribute 

ROUND_NAME = 'Internet Freedom Fund (archive round)'

APPLICATION_TYPE class-attribute instance-attribute 

APPLICATION_TYPE = 'proposal'

STREAMFIELD_MAP class-attribute instance-attribute 

STREAMFIELD_MAP = {'field_pr_confidentiality': {'id': '65fb2c22-a0c5-4cde-94a7-feb27072bc3d', 'type': 'boolean'}, 'field_pr_conflicts': {'id': 'dd75ce49-e3c4-43da-b724-4cb8bb88dcf8', 'type': 'map', 'map': {'0': 'No', '1': 'Yes'}}, 'field_pr_disclosure': {'id': '9f7fe70b-97b5-4263-98ac-a45bf97b59d0', 'type': 'value', 'key': 'safe_value'}, 'field_pr_liked': {'id': 'e91ed603-61ad-483e-be7b-21716d05a3bd', 'type': 'value', 'key': 'safe_value'}, 'field_pr_concern': {'id': '821fb071-7db7-4cc1-ac3a-34b9eee40c94', 'type': 'value', 'key': 'safe_value'}, 'field_pr_red_flags': {'id': '021624ac-6628-430d-ba86-e68fd518c87e', 'type': 'value', 'key': 'safe_value'}, 'field_pr_overview_rate': {'id': '9c5603d5-f897-42fa-8739-5935769c94bd', 'type': 'score'}, 'field_pr_objectives_rate': {'id': '6b748400-fad9-4b31-bb85-e3a53c99f4df', 'type': 'score'}, 'field_pr_strategy_rate': {'id': 'a806a944-1d8a-4904-ace0-acfce5634a50', 'type': 'score'}, 'field_pr_technical_rate': {'id': '512a86a5-ec5b-4d36-9630-90648b5b43e4', 'type': 'score'}, 'field_pr_alternative_rate': {'id': 'd9695d1d-3373-4acf-ada5-3b2593b3a634', 'type': 'score'}, 'field_pr_usability_rate': {'id': 'e43dd4dc-d2fa-493c-9f55-5a126d0e0579', 'type': 'score'}, 'field_pr_sustainability_rate': {'id': 'ee7009b8-ad18-46b5-a981-ccc52972c0a5', 'type': 'score'}, 'field_pr_collaboration_rate': {'id': 'dc5dc5e0-e4d6-462f-8296-a0e58933e701', 'type': 'score'}, 'field_pr_realism_rate': {'id': '31e9b202-24b1-4993-80b7-9851624e2162', 'type': 'score'}, 'field_pr_qualifications_rate': {'id': 'd3f5479c-68da-41d9-a266-130d383bab6b', 'type': 'score'}, 'field_pr_evaluation_rate': {'id': '2a61c71a-74f6-4963-8850-9289e852f604', 'type': 'score'}, 'field_pr_rationale_rate': {'id': '0d1bf533-968c-44b9-bb30-d437ae039474', 'type': 'score'}, 'field_pr_overview': {'id': '9c5603d5-f897-42fa-8739-5935769c94bd', 'type': 'placeholder'}, 'field_pr_objectives': {'id': '6b748400-fad9-4b31-bb85-e3a53c99f4df', 'type': 'placeholder'}, 'field_pr_strategy': {'id': 'a806a944-1d8a-4904-ace0-acfce5634a50', 'type': 'placeholder'}, 'field_pr_technical': {'id': '512a86a5-ec5b-4d36-9630-90648b5b43e4', 'type': 'placeholder'}, 'field_pr_alternative': {'id': 'd9695d1d-3373-4acf-ada5-3b2593b3a634', 'type': 'placeholder'}, 'field_pr_usability': {'id': 'e43dd4dc-d2fa-493c-9f55-5a126d0e0579', 'type': 'placeholder'}, 'field_pr_sustainability': {'id': 'ee7009b8-ad18-46b5-a981-ccc52972c0a5', 'type': 'placeholder'}, 'field_pr_collaboration': {'id': 'dc5dc5e0-e4d6-462f-8296-a0e58933e701', 'type': 'placeholder'}, 'field_pr_realism': {'id': '31e9b202-24b1-4993-80b7-9851624e2162', 'type': 'placeholder'}, 'field_pr_qualifications': {'id': 'd3f5479c-68da-41d9-a266-130d383bab6b', 'type': 'placeholder'}, 'field_pr_evaluation': {'id': '2a61c71a-74f6-4963-8850-9289e852f604', 'type': 'placeholder'}, 'field_pr_rationale': {'id': '0d1bf533-968c-44b9-bb30-d437ae039474', 'type': 'placeholder'}, 'field_pr_recommendation_comments': {'id': 'comments', 'type': 'value', 'key': 'safe_value'}}

REQUEST_QUESTION_MAP class-attribute instance-attribute 

REQUEST_QUESTION_MAP = {'3618': {0: 'Do the Goals and principles of the application align with the program?', 1: 'Does the application propose a unique contribution to the relevant field?', 2: 'Do you consider the application reasonable and realistic?', 3: 'General Comments'}, '3681': {0: 'What are the positive aspects of this application?', 1: 'What are the negative aspects of this application?', 2: 'What items must the applicant address, if any?'}}

add_arguments 

add_arguments(parser)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
80
81
82
83
def add_arguments(self, parser):
    parser.add_argument(
        "source", type=argparse.FileType("r"), help="Migration source JSON file"
    )

handle 

handle(*args, **options)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
@transaction.atomic
def handle(self, *args, **options):
    with options["source"] as json_data:
        self.data = json.load(json_data)

        # A user can only submitt a single review in the new system so pick the latest one.
        blacklist = {
            "7574",
            "7413",
            "7412",
            "5270",
            "6468",
            "6436",
            "5511",
            "3490",
            "2840",
            "2837",
            "2737",
            "9362",
            "9203",
            "8958",
            "13313",
            "4868",
            "4867",
            "5863",
            "5770",
            "3962",
            "3747",
        }

        counter = 0
        for id in self.data:
            if id not in blacklist:
                self.process(id)
                counter += 1

        self.stdout.write(f"Imported {counter} reviews.")

process 

process(id)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
def process(self, id):
    node = self.data[id]

    try:
        review = Review.objects.get(drupal_id=node["nid"])
    except Review.DoesNotExist:
        review = Review(drupal_id=node["nid"])

    # Disable auto_* on date fields so imported dates are used.
    for field in review._meta.local_fields:
        if field.name == "created_at":
            field.auto_now_add = False
        elif field.name == "updated_at":
            field.auto_now = False

    # TODO timezone?
    review.created_at = datetime.fromtimestamp(int(node["created"]), timezone.utc)
    review.updated_at = datetime.fromtimestamp(int(node["changed"]), timezone.utc)
    review.author = self.get_user(node["uid"])
    review.recommendation = self.get_recommendation(node)
    review.submission = self.get_submission(node)
    review.revision = review.submission.live_revision

    if self.CONTENT_TYPE == "fund":
        ROUND = Round.objects.get(title=self.ROUND_NAME)
        if self.APPLICATION_TYPE == "request":
            FORM = RoundBaseReviewForm.objects.get(round=ROUND)
        elif self.APPLICATION_TYPE == "concept":
            FORM = RoundBaseReviewForm.objects.filter(round=ROUND)[0]
        elif self.APPLICATION_TYPE == "proposal":
            FORM = RoundBaseReviewForm.objects.filter(round=ROUND)[1]
        review.form_fields = FORM.form.form_fields
    elif self.CONTENT_TYPE == "lab":
        LAB = LabType.objects.get(title=self.LAB_NAME)
        FORM = LabBaseReviewForm.objects.get(lab=LAB)
        review.form_fields = FORM.form.form_fields

    form_data = {}

    for field in node:
        if field in self.STREAMFIELD_MAP:
            try:
                id = self.STREAMFIELD_MAP[field]["id"]
                form_data[id] = self.get_field_value(field, node)
            except TypeError:
                pass

    if "comments" not in form_data or not form_data["comments"]:
        form_data["comments"] = "No comment."

    form_data["recommendation"] = review.recommendation

    review.form_data = form_data

    scores = [
        review.data(field.id)[1]
        for field in review.score_fields
        if review.data(field.id)[1] != NA
    ]
    try:
        review.score = sum(scores) / len(scores)
    except ZeroDivisionError:
        review.score = 0

    try:
        review.save()
        self.stdout.write(f"Processed \"{node['title']}\" ({node['nid']})")
    except IntegrityError:
        self.stdout.write(
            f"*** Skipped \"{node['title']}\" ({node['nid']}) due to IntegrityError"
        )

get_user 

get_user(uid)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
195
196
197
198
199
200
def get_user(self, uid):
    try:
        User = get_user_model()
        return User.objects.get(drupal_id=uid)
    except User.DoesNotExist:
        return None

get_field_value 

get_field_value(field, node)

Handles the following formats: field: {(safe_)value: VALUE} field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities. field: {tid: ID} -- or term ID. fk to Categories field: [] field: [{value|target_id|tid: VALUE},]

Source code in hypha/apply/review/management/commands/migration_review_base.py 
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
def get_field_value(self, field, node):
    """
    Handles the following formats:
    field: {(safe_)value: VALUE}
    field: {target_id: ID} -- Drupal ForeignKey. Reference to other node or user entities.
    field: {tid: ID} -- or term ID. fk to Categories
    field: []
    field: [{value|target_id|tid: VALUE},]
    """
    mapping = self.STREAMFIELD_MAP[field]
    mapping_type = mapping["type"]
    key = mapping.get("key", "value")
    source_value = node[field]
    value = None

    if mapping_type == "direct":
        value = source_value
    elif mapping_type == "value":
        if key in source_value:
            value = self.nl2br(source_value[key]) if source_value else ""
        else:
            value = self.nl2br(source_value["value"]) if source_value else ""
    elif mapping_type == "merge_value":
        values = []
        i = 0
        for item in source_value:
            question = self.REQUEST_QUESTION_MAP[node["request_nid"]]
            if i in question:
                values.append(f"<strong>{question[i]}</strong>{item[key]}<br>\n")
            i += 1
        merged_values = "".join(values)
        value = self.nl2br(merged_values) if source_value else ""
    elif mapping_type == "score":
        value_rate = int(source_value[key]) if source_value else NA
        value_text = ""
        if "_rate" in field:
            field = field.replace("field_field_", "field_")
            text_field = field[:-5]
            if text_field in self.STREAMFIELD_MAP:
                value_text_field = node[text_field]
                if "safe_value" in value_text_field:
                    value_text = (
                        self.nl2br(value_text_field["safe_value"])
                        if value_text_field
                        else ""
                    )
                else:
                    value_text = (
                        self.nl2br(value_text_field["value"])
                        if value_text_field
                        else ""
                    )
        value = [value_text, value_rate]
    elif mapping_type == "map" and "map" in "mapping":
        value = mapping["map"].get(source_value[key])
    elif mapping_type == "boolean":
        value = source_value[key] == "1" if source_value else False

    return value

get_recommendation 

get_recommendation(node)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
262
263
264
265
266
267
268
269
def get_recommendation(self, node):
    mapping = self.REVIEWFIELD_MAP[node["type"]]
    field_name = mapping["recommendation"]
    rec_map = mapping.get("rec_map")
    try:
        return rec_map[node[field_name]["value"]]
    except TypeError:
        return 0

get_submission 

get_submission(node)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
271
272
273
274
275
276
277
278
def get_submission(self, node):
    mapping = self.REVIEWFIELD_MAP[node["type"]]
    field_name = mapping["submission"]
    try:
        nid = node[field_name]["target_id"]
        return ApplicationSubmission.objects.get(drupal_id=nid)
    except ApplicationSubmission.DoesNotExist:
        return None

nl2br 

nl2br(value)
Source code in hypha/apply/review/management/commands/migration_review_base.py 
280
281
def nl2br(self, value):
    return value.replace("\r\n", "<br>\n")