review-automation-server/name_extractor/utils.py

import re
from decimal import Decimal
from django.conf import settings
from .models import Staff

from review.models import Review
from facebook_app.models import FacebookReview, FacebookPage
from yelp.models import YelpReview, YelpLocation

ner_model = getattr(settings, 'SPACY_NER_MODEL')


def get_all_names(text):
    doc = ner_model(text)
    names = {ent.text for ent in doc.ents if ent.label_ in ['PERSON', 'ORG', 'PRODUCT']}
    return names


def all_staffs(location_id):
    names = []
    staffs = Staff.objects.filter(location_id=location_id)
    for s in staffs:
        names.extend(s.get_nick_names)
    return set(names)


def clean_text(text):
    # replace some letter in text for getting better performance
    text = re.sub(r':\s*', ' ', text)
    text = re.sub(r'&', ', ', text)
    text = re.sub(r'/', ', ', text)
    text = re.sub(r'\.*\n\.*', '.', text)
    text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text)
    return text


def extract_names(text, location_id):
    names = get_all_names(text)
    cleaned_names = []
    for name in names:
        for n in name.split():
            cleaned_names.append(n) if n.lower() in all_staffs(location_id) else None
    return cleaned_names


def add_point_to_staff_profile(text, location_id, platform):
    staffs = Staff.objects.filter(location_id=location_id)
    names = extract_names(text, location_id)
    point_unit = Decimal(1/len(names)) if not names == [] else 0

    for name in names:
        for staff in staffs:
            if name.lower() in staff.get_nick_names:
                if platform == 'google':
                    staff.name_mentioned_google += 1
                elif platform == 'facebook':
                    staff.name_mentioned_facebook += 1
                elif platform == 'yelp':
                    staff.name_mentioned_yelp += 1
                else:
                    raise AttributeError('No such platform found.')
                staff.total_units += point_unit
                staff.save()


def make_all_staffs_point_zero():
    staffs = Staff.objects.all()
    for s in staffs:
        s.total_units = 0
        s.name_mentioned_google = 0
        s.name_mentioned_facebook = 0
        s.name_mentioned_yelp = 0
        s.save()


def extract_names_from_reviews(start_date, end_date, location_id=None):
    # For google reviews
    google_reviews = Review.objects.filter(
        location_id=location_id,
        create_time__range=(start_date, end_date)
    ).exclude(comment=None)

    for review in google_reviews:
        add_point_to_staff_profile(text=review.comment, location_id=location_id, platform='google')

    # For facebook review
    facebook_reviews = FacebookReview.objects.filter(
        page__location__location_id=location_id,
        create_time__range=(start_date, end_date)
    ).exclude(review_text=None)

    for review in facebook_reviews:
        add_point_to_staff_profile(text=review.review_text, location_id=location_id, platform='facebook')

    # For yelp reviews
    yelp_reviews = YelpReview.objects.filter(
        location__location_id=location_id,
        date_posted__range=(start_date, end_date)
    ).exclude(comment=None)

    for review in yelp_reviews:
        add_point_to_staff_profile(text=review.comment, location_id=location_id, platform='yelp')