review-automation-server/nlu_job/nlu_utils.py

import re
from django.conf import settings
from requests import post
import json
from difflib import SequenceMatcher
from review.models import Review

nlu_server_url = settings.NLU_SERVER_URI
ner_model = settings.MODEL


def filter_with_last_ten_reviews(location_id, replies):
    replies = list(replies)
    revs = Review.objects.filter(location_id=location_id).exclude(reply=None).order_by('-update_time')[:10]
    for r in revs:
        s1 = r.reply.replied_text
        for rep in replies:
            s2 = rep.reply
            similarity = SequenceMatcher(None, s1, s2).ratio()
            if similarity > 0.7:
                replies.remove(rep)
                print('%.2f'%similarity, ' -------------- ', rep.reply_category)

    return replies


def clean_text(text):
    # replace some letter in text for getting better performance
    text = re.sub(r':\s*', ' ', text)
    text = re.sub(r'&', ', ', text)
    text = re.sub(r'/', ', ', text)
    text = re.sub(r'\.*\n\.*', ', ', text)
    text = re.sub(r'[dD][rR](\.|\s*)*', 'Dr. ', text)
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               "]+", flags=re.UNICODE)
    text = re.sub(emoji_pattern, ' ', text)
    return text


def model_inference(text):
    url = nlu_server_url + '/model/parse'
    text = clean_text(text)
    payload = {'text': text}
    headers = {'content-type': 'application/json'}

    response = post(url, data=json.dumps(payload), headers=headers)
    if response.status_code == 200:
        return response.json()
    return response


def is_a_name(name):
    '''
    function that decide whether it is a person name or not
    :param -> a string usually reviewer name:
    :return -> a boolean True/False:
    '''

    response = model_inference(name.title())
    entities = response.get('entities')
    if not entities:
        return False
    entity = entities[0]
    if entity.get('entity') == 'PERSON':
        return True
    else:
        return False


def analyze_inference(response):
    '''
    response has four property
    ['intent', 'entities', 'intent_ranking', 'text']
    we took all intents that has more than 10% of intent confident.
    all the intents that has bellow confidence has been omitted.
    :param response: JSON -> a json response that RASA NLU server respond.
    :return: DICT ->dictionary with key of intent and value of it's confident.
    '''

    res_intents = response.get('intent_ranking')
    intents = {}
    for intent in res_intents:
        key = intent.get('name')
        values = intent.get('confidence')
        if values > 0.1:
            intents[key] = int(values*100)

    return intents


def name_entity_recognition(text):
    doc = ner_model(text)
    names = [n for n in doc.ents if n.label_ == 'PERSON']
    return names