store all reviews by calling a centralized function

gauth/auth_utils.py

@@ -16,7 +16,6 @@ def get_access_token(request):
     if 'credentials' in request.session and not has_expired(request.session['credentials']):
         cred = request.session['credentials']
         return cred['access_token']
-
     access_token, expires_in = refresh_access_token()
     expired_at = timezone.datetime.now() + timezone.timedelta(seconds=expires_in)
     expiry = str(expired_at)
@@ -63,3 +62,9 @@ def get_google_account_id(access_token):
     accounts_name = res['accounts'][0]['name']
     id = accounts_name.split('/')[-1]
     return id
+
+
+def get_gmb_id():
+    user = User.objects.get(username='admin@ercare')
+    gmb_id = UserModel.objects.get(user=user).gmb_account_id
+    return user, gmb_id

gauth/location_utils.py

@@ -1,7 +1,9 @@
-from django.contrib.auth.models import User
 from requests import get
-from .auth_utils import refresh_access_token
-from .models import UserModel, Location
+from .models import Location
+from django.conf import settings
+from .auth_utils import get_gmb_id
+
+ACCESS_TOKEN_URI = settings.HOST_URI + '/token'
 
 
 def get_all_location_ids():
@@ -11,14 +13,14 @@ def get_all_location_ids():
     There is a location which we don't need to add into database.
     It is a Medical Billing Service, That's why it is nothing to do with review. 
     '''
-    ids.remove('5397588228065547694')
+    if '5397588228065547694' in ids:
+        ids.remove('5397588228065547694')
     return ids
 
 
 def populate_locations():
-    access_token, _ = refresh_access_token()
-    user = User.objects.get(username='admin@ercare')
-    gmb_id = UserModel.objects.get(user=user).gmb_account_id
+    access_token = get(ACCESS_TOKEN_URI).text
+    user, gmb_id = get_gmb_id()
     url = 'https://mybusiness.googleapis.com/v4/accounts/' + gmb_id + '/locations'
     headers = {
         'authorization': 'Bearer ' + access_token,
@@ -30,10 +32,14 @@ def populate_locations():
         # loc['name'] = 'accounts/103266181421855655295/locations/8916258876770296726'
         loc_id = loc['name'].split('/')[-1]
         location_ids = get_all_location_ids()
+        location_ids.append('5397588228065547694')
         if loc_id in location_ids:
             continue
-        loc_name = loc['locationName']
+        loc_name = loc['address']['locality']
         loc_website = loc['websiteUrl']
         loc_display = loc['primaryCategory']['displayName']
         location = Location.objects.create(location_id=loc_id, location_name=loc_name, website_url=loc_website, display_name=loc_display, owner=user)
         location.save()
+
+
+        # loc_display = loc_website.spilt('/')[-1]

gauth/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0 on 2019-12-26 13:17
+# Generated by Django 3.0 on 2019-12-30 10:39
 
 from django.conf import settings
 from django.db import migrations, models
@@ -18,9 +18,19 @@ class Migration(migrations.Migration):
             name='UserModel',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('refresh_token', models.CharField(max_length=30)),
-                ('gmb_account_id', models.CharField(max_length=30)),
+                ('refresh_token', models.CharField(blank=True, max_length=30, null=True)),
+                ('gmb_account_id', models.CharField(blank=True, max_length=30, null=True)),
                 ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='token_of', to=settings.AUTH_USER_MODEL)),
             ],
         ),
+        migrations.CreateModel(
+            name='Location',
+            fields=[
+                ('location_id', models.CharField(max_length=50, primary_key=True, serialize=False)),
+                ('location_name', models.CharField(max_length=50)),
+                ('website_url', models.URLField()),
+                ('display_name', models.CharField(max_length=50)),
+                ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
     ]

gauth/migrations/0002_auto_20191229_0659.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.0 on 2019-12-29 06:59
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('gauth', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='usermodel',
-            name='gmb_account_id',
-            field=models.CharField(blank=True, max_length=30, null=True),
-        ),
-        migrations.AlterField(
-            model_name='usermodel',
-            name='refresh_token',
-            field=models.CharField(blank=True, max_length=30, null=True),
-        ),
-    ]

gauth/migrations/0002_auto_20191231_1139.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2019-12-31 11:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gauth', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='usermodel',
+            name='refresh_token',
+            field=models.CharField(blank=True, max_length=60, null=True),
+        ),
+    ]

gauth/migrations/0003_auto_20191231_1140.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2019-12-31 11:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gauth', '0002_auto_20191231_1139'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='usermodel',
+            name='refresh_token',
+            field=models.CharField(blank=True, max_length=20, null=True),
+        ),
+    ]

gauth/migrations/0003_location.py

@@ -1,27 +0,0 @@
-# Generated by Django 3.0 on 2019-12-29 09:56
-
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('gauth', '0002_auto_20191229_0659'),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='Location',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('location_id', models.CharField(max_length=50)),
-                ('location_name', models.CharField(max_length=50)),
-                ('website_url', models.URLField()),
-                ('display_name', models.CharField(max_length=50)),
-                ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-            ],
-        ),
-    ]

gauth/migrations/0004_auto_20191229_1033.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.0 on 2019-12-29 10:33
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('gauth', '0003_location'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='location',
-            name='location_id',
-            field=models.CharField(max_length=50, unique=True),
-        ),
-    ]

gauth/migrations/0004_auto_20191231_1144.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2019-12-31 11:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gauth', '0003_auto_20191231_1140'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='usermodel',
+            name='refresh_token',
+            field=models.CharField(blank=True, max_length=120, null=True),
+        ),
+    ]

gauth/migrations/0005_auto_20200101_0803.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0 on 2020-01-01 08:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gauth', '0004_auto_20191231_1144'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='location',
+            name='display_name',
+            field=models.CharField(max_length=120),
+        ),
+        migrations.AlterField(
+            model_name='location',
+            name='location_name',
+            field=models.CharField(max_length=120),
+        ),
+    ]

gauth/models.py

@@ -4,7 +4,7 @@ from django.contrib.auth.models import User
 
 class UserModel(models.Model):
     user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='token_of')
-    refresh_token = models.CharField(max_length=30, null=True, blank=True)
+    refresh_token = models.CharField(max_length=120, null=True, blank=True)
     gmb_account_id = models.CharField(max_length=30, null=True, blank=True)
 
     def __str__(self):
@@ -12,10 +12,10 @@ class UserModel(models.Model):
 
 
 class Location(models.Model):
-    location_id = models.CharField(max_length=50, unique=True)
-    location_name = models.CharField(max_length=50)
+    location_id = models.CharField(max_length=50, primary_key=True)
+    location_name = models.CharField(max_length=120)
     website_url = models.URLField()
-    display_name = models.CharField(max_length=50)
+    display_name = models.CharField(max_length=120)
     owner = models.ForeignKey(User, on_delete=models.CASCADE, null=True)
 
     def __str__(self):

gauth/urls.py

@@ -1,8 +1,8 @@
 from django.urls import path
-from .views import google_auth, authorize, oauth2callback, index
+from .views import google_auth, authorize, oauth2callback, get_token
 
 urlpatterns = [
-    path('', index, name='home'),
+    path('token', get_token, name='token'),
     path('gmb-authenticate/', google_auth, name='gmb-auth'),
     path('authorize', authorize, name='authorize'),
     path('oauth2callback', oauth2callback, name='oauth2callback'),

gauth/views.py

@@ -18,7 +18,7 @@ flow = google_auth_oauthlib.flow.Flow.from_client_secrets_file(
         redirect_uri="http://127.0.0.1:8000/oauth2callback")
 
 
-def index(request):
+def get_token(request):
     get_access_token(request)
     cred = request.session['credentials']
     return HttpResponse(cred['access_token'])
@@ -33,7 +33,7 @@ def google_auth(request):
 
 
 def authorize(request):
-    authorization_url, state = flow.authorization_url(access_type='offline')
+    authorization_url, state = flow.authorization_url(access_type='offline', include_granted_scopes='true')
 
     # Store the state so the callback can verify the auth server response.
     request.session['state'] = state
@@ -47,7 +47,6 @@ def oauth2callback(request):
     flow.state = state
     code = request.GET.get('code', False)
     flow.fetch_token(code=code)
-
     credentials = flow.credentials
     account_id = get_google_account_id(credentials.token)
     user = User.objects.filter(username='admin@ercare').first()
@@ -56,7 +55,7 @@ def oauth2callback(request):
     user_model.gmb_account_id = account_id
     user_model.save()
     request.session['credentials'] = credentials_to_dict(credentials)
-    return redirect(reverse('home'))
+    return redirect(reverse('token'))
 
 
 def credentials_to_dict(credentials):

review/admin.py

@@ -1,3 +1,10 @@
 from django.contrib import admin
+from .models import Review, Reply
 
-# Register your models here.
+
+class ReviewAdmin(admin.ModelAdmin):
+    list_display = ('reviewer_name', 'star_rating', 'comment')
+
+
+admin.site.register(Review, ReviewAdmin)
+admin.site.register(Reply)

review/csv_exporter.py

@@ -0,0 +1,15 @@
+from .models import Review
+import csv
+revs = Review.objects.all()
+
+
+def write_to_csv():
+    with open('review_dataset.csv', 'w') as file:
+        header_row = ['review', 'reply']
+        writer = csv.writer(file)
+        writer.writerow(header_row)
+        for rev in revs:
+            review = rev.comment if rev.comment else 'NULL'
+            reply = rev.reply.replied_text if rev.reply else 'NULL'
+            row = [review, reply]
+            writer.writerow(row)

review/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0 on 2019-12-26 13:12
+# Generated by Django 3.0 on 2019-12-30 10:39
 
 from django.db import migrations, models
 import django.db.models.deletion
@@ -9,19 +9,10 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
+        ('gauth', '0001_initial'),
     ]
 
     operations = [
-        migrations.CreateModel(
-            name='Location',
-            fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('location_id', models.CharField(max_length=50)),
-                ('location_name', models.CharField(max_length=50)),
-                ('website_url', models.URLField()),
-                ('display_name', models.CharField(max_length=50)),
-            ],
-        ),
         migrations.CreateModel(
             name='Reply',
             fields=[
@@ -33,16 +24,15 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='Review',
             fields=[
-                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('review_id', models.CharField(max_length=50)),
+                ('review_id', models.CharField(max_length=50, primary_key=True, serialize=False)),
                 ('comment', models.CharField(max_length=1000)),
                 ('create_time', models.DateTimeField()),
                 ('update_time', models.DateTimeField()),
                 ('star_rating', models.IntegerField()),
                 ('reviewer_name', models.CharField(max_length=50)),
                 ('reviewer_photo', models.URLField()),
-                ('location', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='review.Location')),
-                ('reply', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='review.Reply')),
+                ('location', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='gauth.Location')),
+                ('reply', models.OneToOneField(null=True, on_delete=django.db.models.deletion.CASCADE, to='review.Reply')),
             ],
         ),
     ]

review/migrations/0002_auto_20191229_0603.py

@@ -1,26 +0,0 @@
-# Generated by Django 3.0 on 2019-12-29 06:03
-
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-        ('review', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='location',
-            name='owner',
-            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='review',
-            name='location',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='review.Location'),
-        ),
-    ]

review/migrations/0002_auto_20191231_1012.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2019-12-31 10:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='review',
+            name='comment',
+            field=models.CharField(max_length=1000, null=True),
+        ),
+    ]

review/migrations/0003_auto_20191229_0956.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.0 on 2019-12-29 09:56
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('gauth', '0003_location'),
-        ('review', '0002_auto_20191229_0603'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='review',
-            name='location',
-            field=models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='gauth.Location'),
-        ),
-        migrations.DeleteModel(
-            name='Location',
-        ),
-    ]

review/migrations/0003_auto_20200101_0803.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0 on 2020-01-01 08:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0002_auto_20191231_1012'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='review',
+            name='comment',
+            field=models.CharField(max_length=2500, null=True),
+        ),
+    ]

review/migrations/0004_auto_20200101_0807.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0 on 2020-01-01 08:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0003_auto_20200101_0803'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='review',
+            name='review_id',
+            field=models.CharField(max_length=150, primary_key=True, serialize=False),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='reviewer_name',
+            field=models.CharField(max_length=100),
+        ),
+    ]

review/migrations/0005_auto_20200101_0830.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0 on 2020-01-01 08:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0004_auto_20200101_0807'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='reply',
+            name='replied_text',
+            field=models.CharField(max_length=2500),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='comment',
+            field=models.CharField(max_length=8000, null=True),
+        ),
+    ]

review/migrations/0006_auto_20200101_0847.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0 on 2020-01-01 08:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('review', '0005_auto_20200101_0830'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='reply',
+            name='replied_text',
+            field=models.TextField(),
+        ),
+        migrations.AlterField(
+            model_name='review',
+            name='comment',
+            field=models.TextField(null=True),
+        ),
+    ]

review/models.py

@@ -4,7 +4,7 @@ from gauth.models import Location
 
 
 class Reply(models.Model):
-    replied_text = models.CharField(max_length=500)
+    replied_text = models.TextField()
     create_time = models.DateTimeField()
 
     def __str__(self):
@@ -12,15 +12,15 @@ class Reply(models.Model):
 
 
 class Review(models.Model):
-    review_id = models.CharField(max_length=50)
-    comment = models.CharField(max_length=1000)
+    review_id = models.CharField(max_length=150, primary_key=True)
+    comment = models.TextField(null=True)
     create_time = models.DateTimeField()
     update_time = models.DateTimeField()
     star_rating = models.IntegerField()
-    reviewer_name = models.CharField(max_length=50)
+    reviewer_name = models.CharField(max_length=100)
     reviewer_photo = models.URLField()
     location = models.ForeignKey(Location, on_delete=models.DO_NOTHING)
-    reply = models.OneToOneField(Reply, on_delete=models.CASCADE)
+    reply = models.OneToOneField(Reply, on_delete=models.CASCADE, null=True)
 
     def __str__(self):
         return f'{self.reviewer_name} - {self.comment}'

review/review_utils.py

@@ -0,0 +1,107 @@
+from requests import get
+from django.conf import settings
+from gauth.auth_utils import get_gmb_id
+from gauth.location_utils import get_all_location_ids
+from django.db.models import Max
+from .models import Review, Reply
+from gauth.models import Location
+
+
+ACCESS_TOKEN_URI = settings.HOST_URI + '/token'
+
+STAR_REVIEW_NUM = {'STAR_RATING_UNSPECIFIED': 0, 'ONE': 1, 'TWO': 2, 'THREE': 3, 'FOUR': 4, 'FIVE': 5}
+USER, GMB_ID = get_gmb_id()
+
+
+def get_review_list_url(loc_id, next_page_token=''):
+    # An helper function that make a url that need to consume GMB review api
+    return 'https://mybusiness.googleapis.com/v4/accounts/' + GMB_ID + '/locations/' + loc_id + '/reviews?pageToken='+next_page_token
+
+
+def get_max_date(loc_id):
+    # Function that takes a location id and return the latest updated review.
+    largest = Review.objects.filter(location_id=loc_id).aggregate(Max('create_time'))['create_time__max']
+    max_date = largest if largest else '1970-01-11 17:41:17.532740'
+    return max_date
+
+
+def get_auth_header():
+    access_token = get(ACCESS_TOKEN_URI).text
+    headers = {
+        'authorization': 'Bearer ' + access_token,
+        'content-type': 'application/json'
+    }
+    return headers
+
+
+def filter_unrecorded_review_by_date(reviews, max_date):
+    # A function that return only those reviews whose has larger value than
+    # the max create_time value in review database.
+    filtered_reviews = []
+    for rev in reviews:
+        if rev['createTime'] >= str(max_date):
+            filtered_reviews.append(rev)
+    return filtered_reviews
+
+
+def insert_review_into_database(unrecorded_reviews, loc):
+    for review in unrecorded_reviews:
+        review_id = review.get('reviewId')
+        # Check the review already in database then We don't need to store again.
+        rev = Review.objects.filter(pk=review_id).first()
+        if rev:
+            continue
+        comment = review.get('comment')
+        create_time = review.get('createTime')
+        update_time = review.get('updateTime')
+        star_rating = STAR_REVIEW_NUM[review.get('starRating')]
+        reviewer = review.get('reviewer')
+        reviewer_name = reviewer.get('displayName')
+        reviewer_photo = reviewer.get('profilePhotoUrl')
+        review_reply = review.get('reviewReply')
+        if review_reply:
+            replied_text = review_reply.get('comment')
+            create_time = review_reply.get('updateTime')
+            reply = Reply.objects.create(replied_text=replied_text, create_time=create_time)
+        else:
+            reply = None
+
+        review = Review(
+            review_id=review_id,
+            comment=comment,
+            create_time=create_time,
+            update_time=update_time,
+            star_rating=star_rating,
+            reviewer_name=reviewer_name,
+            reviewer_photo=reviewer_photo,
+            location=loc,
+            reply=reply
+        )
+        review.save()
+
+
+def fetch_all_review(loc_id):
+    loc = Location.objects.get(pk=loc_id)
+    max_date = get_max_date(loc_id)
+    next_page_token = ''
+    headers = get_auth_header()
+    while True:
+        url = get_review_list_url(loc_id, next_page_token)
+        res = get(url, headers=headers)
+        if res.status_code == 401:
+            headers = get_auth_header()
+            continue
+        data = res.json()
+        reviews = data['reviews']
+        unrecorded_reviews = filter_unrecorded_review_by_date(reviews, max_date)
+        if len(unrecorded_reviews) != 0:
+            insert_review_into_database(unrecorded_reviews, loc)
+        next_page_token = data.get('nextPageToken')
+        if next_page_token is None:
+            break
+
+
+def populate_reviews():
+    locations = get_all_location_ids()
+    for loc_id in locations:
+        fetch_all_review(loc_id)

review_automation/settings.py

@@ -62,12 +62,17 @@ WSGI_APPLICATION = 'review_automation.wsgi.application'
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'ENGINE': 'django.db.backends.mysql',
+        'NAME': 'review_automation',
+        'USER': 'root',
+        'PASSWORD': 'sad2002S1',
+        'HOST': 'localhost',
+        'PORT': '3306',
     }
 }
 
 
+
 # Password validation
 
 AUTH_PASSWORD_VALIDATORS = [
@@ -112,3 +117,4 @@ LOGIN_REDIRECT_URL = '/'
 CLIENT_ID = "174657415928-0bt50gt42pslq47gf21ao67n15rom96r.apps.googleusercontent.com"
 CLIENT_SECRET = "ZXYpt07Su0pW3y3jPGOXY_C_"
 TOKEN_URI = "https://oauth2.googleapis.com/token"
+HOST_URI = "http://127.0.0.1:8000"