added batch network call and cleaner of comment

review/review_utils.py

@@ -1,24 +1,37 @@
+import re
 import json
-from requests import get, put
+from requests import get, put, post
 from gauth.auth_utils import get_gmb_id, get_auth_header
 from .models import Review, Reply
 from gauth.models import Location
 
 from django.utils import timezone
 
-
+_, account_id = get_gmb_id()
 STAR_REVIEW_NUM = {'STAR_RATING_UNSPECIFIED': 0, 'ONE': 1, 'TWO': 2, 'THREE': 3, 'FOUR': 4, 'FIVE': 5}
+BASE_URL = f'https://mybusiness.googleapis.com/v4/'
+
+
+def clean_comment(text):
+    rules = [
+        {r'[^\x00-\x7F]+': ''},
+        {r'^\(Google-\s*\)(.|\n|]\s)*\(\)': ''}
+    ]
+    for rule in rules:
+        for (k, v) in rule.items():
+            regex = re.compile(k)
+            text = regex.sub(v, text)
+    text = text.rstrip()
+    return text
 
 
 def get_review_list_url(location_id, next_page_token=''):
     # An helper function that make a url that need to consume GMB review api
-    _, account_id = get_gmb_id()
-    return f'https://mybusiness.googleapis.com/v4/accounts/{account_id}/locations/{location_id}/reviews?pageToken='+next_page_token
+    return f'{BASE_URL}accounts/{account_id}/locations/{location_id}/reviews?pageToken='+next_page_token
 
 
 def get_reply_url(location_id, review_id):
-    _, account_id = get_gmb_id()
-    return f'https://mybusiness.googleapis.com/v4/accounts/{account_id}/locations/{location_id}/reviews/{review_id}/reply'
+    return f'{BASE_URL}accounts/{account_id}/locations/{location_id}/reviews/{review_id}/reply'
 
 
 def reply_review(review, replied_text):
@@ -38,8 +51,8 @@ def reply_review(review, replied_text):
 def insert_review_into_database(reviews, loc_id):
     '''
     Insert reviews to database.
-    :param unrecorded_reviews: all reviews for location.
-    :param loc: location that unrecorded_reviews belongs to.
+    :param reviews: all reviews for location.
+    :param loc_id: location id unrecorded_reviews belongs to.
     :return: It insert all reviews if it is not exits in database and return nothing.
     '''
     for rev in reviews:
@@ -62,10 +75,11 @@ def insert_review_into_database(reviews, loc_id):
             replied_text = review_reply.get('comment')
             create_time = review_reply.get('updateTime')
 
-            review.reply = Reply.objects.create(
+            reply, created = Reply.objects.update_or_create(
                 replied_text=replied_text,
                 create_time=create_time
                 )
+            review.reply = reply
         else:
             review.reply = None
         review.save()
@@ -120,6 +134,71 @@ def fetch_last_20_reviews(loc_id, page_size=20):
     update_location_data(loc_id, average_rating, total_reviews, total_reviews_db)
 
 
+def store_batch_of_reviews(reviews):
+    for rev in reviews:
+        location_id = rev.get('name').split('/')[-1]
+        rev = rev.get('review')
+
+        review_id = rev.get('reviewId')
+        try:
+            review = Review.objects.get(pk=review_id)
+        except Review.DoesNotExist:
+            review = Review(review_id=review_id)
+        review.comment = rev.get('comment')
+        review.create_time = rev.get('createTime')
+        review.update_time = rev.get('updateTime')
+        review.star_rating = STAR_REVIEW_NUM[rev.get('starRating')]
+        reviewer = rev.get('reviewer')
+        review.reviewer_name = reviewer.get('displayName')
+        review.reviewer_photo = reviewer.get('profilePhotoUrl')
+        review.location_id = location_id
+        review_reply = rev.get('reviewReply')
+        # Check if it is already  replied.
+        if review_reply:
+            replied_text = review_reply.get('comment')
+            create_time = review_reply.get('updateTime')
+
+            reply, created = Reply.objects.update_or_create(
+                replied_text=replied_text,
+                create_time=create_time
+            )
+            review.reply = reply
+        else:
+            review.reply = None
+        review.save()
+
+
+def fetch_batch_of_reviews():
+    headers = get_auth_header()
+    url = f'{BASE_URL}accounts/{account_id}/locations:batchGetReviews'
+    # location names should be in this format:
+    # "accounts/103266181421855655295/locations/8918455867446117794",
+    locations = Location.objects.all()
+    location_names = [f'accounts/{account_id}/locations/{loc.location_id}' for loc in locations]
+    '''
+    post data format:
+    {
+      "locationNames": [
+        string
+      ],
+      "pageSize": integer,                     -> Total number of reviews 
+      "pageToken": string,                     -> If has any to go next page.
+      "orderBy": string,                       -> By-default updateTime desc
+      "ignoreRatingOnlyReviews": boolean       -> Whether to ignore rating-only reviews
+    }
+    '''
+    payload = json.dumps({
+        "locationNames": location_names
+    })
+    response = post(url, headers=headers, data=payload)
+    if response.status_code == 200:
+        data = response.json()
+        location_reviews = data.get('locationReviews')
+        store_batch_of_reviews(location_reviews)
+    else:
+        return None
+
+
 def populate_reviews():
     start = timezone.now()
     locations = Location.objects.all().values('location_id')