final

.gitignore

@@ -0,0 +1,2 @@
+**/*.pyc
+__pycache__

Labs/elasticsearch regrex and filter/esRegFilter.py

@@ -0,0 +1,148 @@
+from elasticsearch import Elasticsearch
+import json
+
+DB_URL = "http://203.252.112.14:9200/" + "nkdb200810"
+es = Elasticsearch(DB_URL)
+# es = Elasticsearch(timeout=30)
+
+body = {}
+
+"""
+    post_Date가 있는 문서 수 확인
+"""
+if False:
+    body["query"] = {"exists": {"field": "post_date"}}
+
+    #count
+    res_isDate = es.count(body)
+    body["query"] = {"match_all": {}}
+    res_all = es.count(body)
+    print(str(res_isDate["count"]) + " docs have post_date field out of " + str(res_all["count"]))
+
+    """
+        nkdb200810
+        post_date가 있는 문서 898 out of 9??
+        없는 문서
+
+        nkdb200811
+        post_date가 있는 문서 9999 out of 13???
+    """
+
+
+
+
+"""
+    regex 적용해보기
+"""
+if False:
+    # body = {"query" : {"regexp" : {"post_date" : {"[0-9]{4}-[0-9]{2}-[0-9]{2}",}}}}
+    # body["query"] = {"regexp": {"post_date": {"value" : "[0-9]{4}-[0-9]{2}-[0-9]{2}",}}}
+    body["query"] = {"regexp": {"post_date": {"value" : "[0-9]{4}",}}}
+    body["_source"] = ["post_date"]
+    body["size"] =  1000
+    # body += json.dumps(query)
+
+
+    """
+        실제... post_date 출력해보기.
+        이제 여기서... regrex 을 적용해보기.
+        app 에서 상원이 로직 가지고 오기
+    """
+
+    #search
+    res = es.search(body)["hits"]["hits"]
+    # print(res[0])
+    # # count_s = 0
+    count = 0
+    for doc in res:
+        data = doc["_source"]["post_date"]
+        print(data)
+        try:
+            print(data)
+            # count += 1
+            # if count > 10:
+            #     break
+        except:
+            count += 1
+            print("ERROR! count : ", count)
+    #     # print(type(date))
+    print("error count : ", count)
+    #     # t = type(date)
+    #     # if date == "string":
+
+
+
+"""
+    합치기 위해서 elasticsaerch aggregation을 사용해보기
+"""
+if False:
+    aggs1 = {"regexp": {"post_date": {"value" : "[0-9]{4}",}}}
+
+    body["aggs"] = {
+        "reg" : aggs1
+    }
+    body["size"] =  1000
+
+
+
+
+    res = es.search(body)
+    print(res)
+    # res = es.search(body)["hits"]["hits"]
+    # count = 0
+    # for doc in res:
+    #     data = doc["_source"]["post_date"]
+    #     print(data)
+    #     try:
+    #         print(data)
+    #         # count += 1
+    #         # if count > 10:
+    #         #     break
+    #     except:
+    #         count += 1
+    #         print("ERROR! count : ", count)
+    # #     # print(type(date))
+    # print("error count : ", count)
+
+
+
+"""
+    regex 적용해보기 with ... 
+"""
+if False:
+    # body = {"query" : {"regexp" : {"post_date" : {"[0-9]{4}-[0-9]{2}-[0-9]{2}",}}}}
+    # body["query"] = {"regexp": {"post_date": {"value" : "[0-9]{4}-[0-9]{2}-[0-9]{2}",}}}
+    body["query"] = {"regexp": {"post_date": {"value" : "[0-9]{4}",}}}
+    body["_source"] = ["post_date"]
+    body["size"] =  1000
+    # body += json.dumps(query)
+
+
+    """
+        실제... post_date 출력해보기.
+        이제 여기서... regrex 을 적용해보기.
+        app 에서 상원이 로직 가지고 오기
+    """
+
+    #search
+    res = es.search(body)["hits"]["hits"]
+    # print(res[0])
+    # # count_s = 0
+    count = 0
+    for doc in res:
+        data = doc["_source"]["post_date"]
+        print(data)
+        try:
+            print(data)
+            # count += 1
+            # if count > 10:
+            #     break
+        except:
+            count += 1
+            print("ERROR! count : ", count)
+    #     # print(type(date))
+    print("error count : ", count)
+    #     # t = type(date)
+    #     # if date == "string":
+
+

Labs/postElasticSearch/postES.py

@@ -1,19 +1,19 @@
 import json
 
-FILE_DIR = "../../raw data sample/rawrawData.json"
+FILE_DIR = "../../raw data sample/rawRawData.json"
 
 with open(FILE_DIR,'r', encoding="utf-8") as fp:
     esData = json.load(fp)
 
 meaningful_data = esData['hits']['hits']
-
+indexName = "frontend_test"
 body = ""
 
 for i,d in enumerate(meaningful_data):
     body +=         json.dumps({'index' : 
     {
-    '_index' : d['_index'],
-    '_type' : d['_type'],
+    '_index' : indexName,
+    '_type' : indexName,
     '_id' : d['_id']
 
     }
@@ -36,14 +36,16 @@
         print(body)
 
 from elasticsearch import Elasticsearch
-import socket
-def get_ip_address():
-    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    s.connect(("8.8.8.8", 80))
-    return s.getsockname()[0]
+
+#function that find current ip address
+# import socket
+# def get_ip_address():
+#     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+#     s.connect(("8.8.8.8", 80))
+#     return s.getsockname()[0]
 
 
 
-DB_URL = "http://localhost:9200/nkdb"
+DB_URL = "localhost:9200/"+indexName
 es = Elasticsearch(DB_URL)
-es.bulk(body)
+es.bulk(body)

Labs/sample user history/history.py

@@ -7,7 +7,7 @@
 import sys
 sys.path.append(str(homeDir))
 
-from common import cmm
+from common import config
 from common import esFunc
 from common import prs
 

Labs/save static to mongo/tfidf2Mng.py

@@ -1,12 +1,12 @@
 import json
-TFIDF_DATA_DIR = "../../../TIBigdataFE/src/assets/entire_tfidf/data.json"
+TFIDF_DATA_DIR = "./tfidfTotaldata.json"
 
 with open(TFIDF_DATA_DIR,'r', encoding="utf-8") as fp:
     tfidfData = json.load(fp)
 
 import pymongo
 from pymongo import MongoClient
 client = MongoClient('localhost',27017)
-db = client.analysis
+db = client.analysis0919
 collection = db.tfidf
 collection.insert_many(tfidfData)