import re import requests import json from decimal import Decimal from django.conf import settings from .models import Staff from review.models import Review from facebook_app.models import FacebookReview, FacebookPage from yelp.models import YelpReview, YelpLocation ner_model_url = getattr(settings, 'NER_SERVER_URI') def get_all_names(text): url = ner_model_url + '/names/all' payload = {'text': text} headers = {'content-type': 'application/json'} res = requests.post(url, data=json.dumps(payload), headers=headers).json() return res.get('names') def all_staffs(location_id): names = [] staffs = Staff.objects.filter(location_id=location_id) for s in staffs: names.extend(s.get_nick_names) return set(names) def clean_text(text): # replace some letter in text for getting better performance text = re.sub(r':\s*', ' ', text) text = re.sub(r'&', ', ', text) text = re.sub(r'/', ', ', text) text = re.sub(r'\.*\n\.*', '.', text) text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text) return text def extract_names(text, location_id): names = get_all_names(text) cleaned_names = [] for name in names: for n in name.split(): cleaned_names.append(n) if n.lower() in all_staffs(location_id) else None return cleaned_names def add_point_to_staff_profile(text, location_id, platform): staffs = Staff.objects.filter(location_id=location_id) names = extract_names(text, location_id) point_unit = Decimal(1/len(names)) if not names == [] else 0 for name in names: for staff in staffs: if name.lower() in staff.get_nick_names: if platform == 'google': staff.name_mentioned_google += 1 elif platform == 'facebook': staff.name_mentioned_facebook += 1 elif platform == 'yelp': staff.name_mentioned_yelp += 1 else: raise AttributeError('No such platform found.') staff.total_units += point_unit staff.save() def make_all_staffs_point_zero(): staffs = Staff.objects.all() for s in staffs: s.total_units = 0 s.name_mentioned_google = 0 s.name_mentioned_facebook = 0 s.name_mentioned_yelp = 0 s.save() def extract_names_from_reviews(start_date, end_date, location_id=None): # For google reviews google_reviews = Review.objects.filter( location_id=location_id, create_time__range=(start_date, end_date) ).exclude(comment=None) for review in google_reviews: add_point_to_staff_profile(text=review.comment, location_id=location_id, platform='google') # For facebook review facebook_reviews = FacebookReview.objects.filter( page__location__location_id=location_id, create_time__range=(start_date, end_date) ).exclude(review_text=None) for review in facebook_reviews: add_point_to_staff_profile(text=review.review_text, location_id=location_id, platform='facebook') # For yelp reviews yelp_reviews = YelpReview.objects.filter( location__location_id=location_id, date_posted__range=(start_date, end_date) ).exclude(comment=None) for review in yelp_reviews: add_point_to_staff_profile(text=review.comment, location_id=location_id, platform='yelp')