NER/script.py at master · mana5a/NER · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
# import spacy
# from spacy import displacy
# from collections import Counter
# import en_core_web_sm
# nlp=en_core_web_sm.load()

# #str='Interest in fuzzy systems was sparked by Seiji Yasunobu and Soji Miyamoto of Hitachi, who in 1985 provided simulations that demonstrated the feasibility of fuzzy control systems for the Sendai Subway'
# doc=nlp(str)
# def ner():
#     print([(X.text,X.label_) for X in doc.ents])
#     return [(X.text,X.label_) for X in doc.ents]

# #ner()
import itertools
import pymongo
import spacy
from spacy import displacy
from collections import Counter
import en_core_web_sm
import json
nlp=en_core_web_sm.load()
import pandas as pd
file=pd.read_csv('placement.csv',skiprows=[0],usecols=[2,3])
df=pd.DataFrame(file)

data=[]
for i in range (0, len(df)):
    obj=dict()
    obj['question']=str(df['question'][i])
    obj['answer']=str(df['answer'][i])
    data.append(obj)

def ner():
    tags = list()
    for i in range(0, len(data)):
        ## uncomment if using python 2.x
        #x = unicode(data[i]['question'], "utf-8")
        doc = nlp(data[i]['question'])
        l = list()
        for ent in doc.ents:
            if(str(ent.label_)==str('CARDINAL') or str(ent.label_)==str('ORDINAL')):
                pass
            else:
                l.append((ent.text, ent.label_))
        tags.append(l)
    x = list()
    for i in range(0, len(data)):
        y = {}
        y['question'] = df['question'][i]
        y['answer'] = df['answer'][i]
        z = list()
        for j in range(0, len(tags[i])):
            #print(type(tags[i][j]))
            z.append(list(tags[i][j]))
        y['tags'] = z
        x.append(y)
    # print(x)
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = myclient["mydatabase2"]
    mycol = mydb["customers"]
    mydb.customers.drop()
    x = mycol.insert_many(x)
    # print("printing flatten")
    flatten = list(itertools.chain.from_iterable(tags))
    # print(flatten)
    # print("flatten done")
    # cursor = mycol.find({})
    # res = list()
    # for i in cursor:
    #     res.append(i)
    return list(set(flatten))

def fetch(args):
    print("args inside", args, len(args), type(args))
    data1 = list()
    data1.append(args)
    print(data1)
    myclient = pymongo.MongoClient("mongodb://localhost:27017/")
    mydb = myclient["mydatabase2"]
    mycol = mydb["customers"]
    # cursor = mycol.find({"tags":{"$elemMatch":{"$elemMatch":{"$in":[args]}}} })
    cursor = mycol.find({'tags':{"$elemMatch":{"$elemMatch":{"$in":data1}}}})

    # print(cursor)

    # cursor = mycol.find({"$text": {"$search": args}})

    data = list()
    for i in cursor:
        print("hello")
        x = dict()
        x['question'] = i['question']
        x['answer'] = i['answer']
        data.append(x)
        print(x)
    print(len(data))
    return list(data)