This commit is contained in:
parent
b70dc0bb54
commit
48dd8d5179
|
|
@ -2,3 +2,5 @@
|
||||||
data/recs.json
|
data/recs.json
|
||||||
|
|
||||||
backup.sql
|
backup.sql
|
||||||
|
|
||||||
|
app/data/
|
||||||
|
|
|
||||||
Binary file not shown.
|
|
@ -5,12 +5,12 @@ import os
|
||||||
from json import dumps
|
from json import dumps
|
||||||
import application.endpoints as endpoints
|
import application.endpoints as endpoints
|
||||||
import application.config as config
|
import application.config as config
|
||||||
from application.db import Session
|
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
api = Api(app, version='1', contact={"name":""}, license={"name":"Online Dienst Dokumentation"}, api_spec_url='/api/swagger')
|
api = Api(app, version='1', contact={"name":""}, license={"name":"Online Dienst Dokumentation"}, api_spec_url='/api/swagger')
|
||||||
|
|
||||||
api.add_resource(endpoints.RecipeList,'/api/v1/recipe/')
|
api.add_resource(endpoints.RecipeList,'/api/v1/recipe/')
|
||||||
|
api.add_resource(endpoints.Images,'/api/v1/images/<string:id>')
|
||||||
|
|
||||||
@app.route("/")
|
@app.route("/")
|
||||||
def index():
|
def index():
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -10,7 +10,7 @@ import time
|
||||||
engine = db.create_engine('mysql+mysqldb://root@server/fs2?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0)
|
engine = db.create_engine('mysql+mysqldb://root@server/fs2?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0)
|
||||||
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
Session = sessionmaker(bind=engine)
|
Session = sessionmaker(bind=engine, autoflush=False)
|
||||||
|
|
||||||
# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object
|
# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object
|
||||||
|
|
||||||
|
|
@ -20,7 +20,7 @@ class Recipe(Base):
|
||||||
name = Column('name', Text)
|
name = Column('name', Text)
|
||||||
instructions = Column('instructions', Text)
|
instructions = Column('instructions', Text)
|
||||||
url = Column('url', Text)
|
url = Column('url', Text)
|
||||||
img = Column('img', LargeBinary)
|
img = Column('img', LargeBinary(length=(2**32)-1))
|
||||||
imgURL = Column('imgURL', Text)
|
imgURL = Column('imgURL', Text)
|
||||||
ingredient = relationship("RecIngred", back_populates="recipe")
|
ingredient = relationship("RecIngred", back_populates="recipe")
|
||||||
|
|
||||||
|
|
@ -30,25 +30,26 @@ class RecIngred(Base):
|
||||||
ingredient_amount = Column('ingredient_amount', Text)
|
ingredient_amount = Column('ingredient_amount', Text)
|
||||||
ingredient_amount_mu = Column('ingredient_amount_mu', Text) # measurement unit
|
ingredient_amount_mu = Column('ingredient_amount_mu', Text) # measurement unit
|
||||||
|
|
||||||
recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'), primary_key=True)
|
|
||||||
ingredient_name = Column(String(50), ForeignKey('ingredient.name'), primary_key=True)
|
|
||||||
|
|
||||||
recipe = relationship("Recipe", back_populates="ingredient")
|
recipe = relationship("Recipe", back_populates="ingredient")
|
||||||
ingredient = relationship("Ingredient", back_populates="recipe")
|
ingredient = relationship("Ingredient", back_populates="recipe")
|
||||||
|
|
||||||
|
recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'), primary_key=True)
|
||||||
|
ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True)
|
||||||
|
|
||||||
|
|
||||||
class Ingredient(Base):
|
class Ingredient(Base):
|
||||||
__tablename__ = "ingredient"
|
__tablename__ = "ingredient"
|
||||||
name = Column('name', String(50), primary_key=True)
|
name = Column('name', String(200), primary_key=True)
|
||||||
recipe = relationship("RecIngred", back_populates="ingredient")
|
recipe = relationship("RecIngred", back_populates="ingredient")
|
||||||
trunks = relationship("IngredTrunk", back_populates="ingredient")
|
trunks = relationship("IngredTrunk", back_populates="ingredient")
|
||||||
|
|
||||||
class IngredTrunk(Base):
|
class IngredTrunk(Base):
|
||||||
__tablename__ = 'ingredtrunk'
|
__tablename__ = 'ingredtrunk'
|
||||||
ingredient_name = Column(String(50), ForeignKey('ingredient.name'), primary_key=True)
|
ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True)
|
||||||
trunk_name = Column(String(50), ForeignKey('ingredient.name'), primary_key=True)
|
trunk_name = Column(String(50), ForeignKey('trunk.name'), primary_key=True)
|
||||||
|
|
||||||
ingredient = relationship("Ingredient", back_populates="trunk")
|
ingredient = relationship("Ingredient", back_populates="trunks")
|
||||||
trunk = relationship("Trunk", back_populates="ingredient")
|
trunk = relationship("Trunk", back_populates="ingredients")
|
||||||
|
|
||||||
class Trunk(Base):
|
class Trunk(Base):
|
||||||
__tablename__ = "trunk"
|
__tablename__ = "trunk"
|
||||||
|
|
@ -5,9 +5,9 @@ import requests
|
||||||
import application.config as config
|
import application.config as config
|
||||||
import json
|
import json
|
||||||
import base64
|
import base64
|
||||||
from application.db import Session, Recipe, Ingredient
|
from application.db2 import Session, Recipe
|
||||||
import search
|
import application.search as search
|
||||||
import migrate
|
import background.migrate as migrate
|
||||||
import time
|
import time
|
||||||
|
|
||||||
class RecipeList(Resource):
|
class RecipeList(Resource):
|
||||||
|
|
@ -20,15 +20,15 @@ class RecipeList(Resource):
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
ingreds = args["ingred"]
|
ingreds = args["ingred"]
|
||||||
|
|
||||||
ingreds = [migrate.stemWord(ingred)[0] for ingred in ingreds + search.defaultArr]
|
ingreds = [migrate.stem(ingred)[0] for ingred in ingreds]
|
||||||
|
|
||||||
start = time.time()
|
start = time.time()
|
||||||
indx = search.fastes(ingreds)
|
indx = search.search2(ingreds)
|
||||||
end = time.time()
|
end = time.time()
|
||||||
print("get recipes",end - start, "\n")
|
print("get recipes",end - start, "\n")
|
||||||
|
|
||||||
start = time.time()
|
#start = time.time()
|
||||||
recs = search.getRecDict(indx, ingreds)
|
recs = search.getRecDict2(indx, ingreds)
|
||||||
end = time.time()
|
end = time.time()
|
||||||
print("calc overlay",end - start, "\n")
|
print("calc overlay",end - start, "\n")
|
||||||
|
|
||||||
|
|
@ -36,5 +36,15 @@ class RecipeList(Resource):
|
||||||
g.session.close()
|
g.session.close()
|
||||||
return flask.make_response(flask.jsonify({'data': recs}), 200)
|
return flask.make_response(flask.jsonify({'data': recs}), 200)
|
||||||
|
|
||||||
|
class Images(Resource):
|
||||||
|
def get(self, id = None):
|
||||||
|
if id is None:
|
||||||
|
flask.make_response(flask.jsonify({'error': "No ID supplied"}), 401)
|
||||||
|
|
||||||
|
session = Session()
|
||||||
|
image = session.query(Recipe.img).filter(Recipe.recipe_id == id).first()[0]
|
||||||
|
image = base64.b64decode(image.encode())
|
||||||
|
return flask.Response(image, mimetype='image/png')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,89 @@
|
||||||
|
|
||||||
|
import application.db2 as db
|
||||||
|
from flask import g
|
||||||
|
import nltk as nltk
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
import time
|
||||||
|
import heapq
|
||||||
|
from collections import Counter
|
||||||
|
import background.migrate
|
||||||
|
|
||||||
|
def search2(inputArr):
|
||||||
|
indx = {}
|
||||||
|
dbSession = db.Session()
|
||||||
|
for inpu in inputArr:
|
||||||
|
x = dbSession.query(db.Trunk.name, db.Recipe.recipe_id).filter(db.Trunk.name == inpu).join(db.IngredTrunk).join(
|
||||||
|
db.Ingredient).join(db.RecIngred).join(db.Recipe).all()
|
||||||
|
|
||||||
|
indx[inpu] = [str(y[1]) for y in x]
|
||||||
|
|
||||||
|
return(indx)
|
||||||
|
|
||||||
|
|
||||||
|
def stemInput(inputArr):
|
||||||
|
inputArr2 = []
|
||||||
|
|
||||||
|
snowball = nltk.SnowballStemmer(language='german')
|
||||||
|
stopset = set(stopwords.words('german'))
|
||||||
|
for word in inputArr:
|
||||||
|
if word in stopset:
|
||||||
|
continue
|
||||||
|
inputArr2.append(snowball.stem(word))
|
||||||
|
return inputArr2
|
||||||
|
#
|
||||||
|
|
||||||
|
def getRecDict2(indx, inputArr):
|
||||||
|
dbSession = db.Session()
|
||||||
|
|
||||||
|
outDict = {}
|
||||||
|
# 2d to 1d
|
||||||
|
indx = sum(indx.values(), [])
|
||||||
|
k = Counter(indx)
|
||||||
|
indx = k.most_common(1000)
|
||||||
|
indx = dict(indx)
|
||||||
|
|
||||||
|
ingred = [x for x in dbSession.query(db.Recipe.recipe_id, db.IngredTrunk.trunk_name, db.IngredTrunk.ingredient_name ).filter(db.Recipe.recipe_id.in_(indx.keys())).join(db.RecIngred).join(db.Ingredient).join(db.IngredTrunk).all()]
|
||||||
|
ingredDict = {}
|
||||||
|
for k,v, i in ingred:
|
||||||
|
if k not in ingredDict:
|
||||||
|
ingredDict[k] = {}
|
||||||
|
if i not in ingredDict[k]:
|
||||||
|
ingredDict[k][i] = []
|
||||||
|
|
||||||
|
ingredDict[k][i].append(v)
|
||||||
|
inputArr += defaultArr
|
||||||
|
for key, value in ingredDict.items():
|
||||||
|
overlay = calcOverlay2(inputArr, value)
|
||||||
|
while overlay in outDict.keys():
|
||||||
|
overlay -= 0.0001
|
||||||
|
outDict[overlay] = int(key)
|
||||||
|
|
||||||
|
outDict2 = {}
|
||||||
|
for key in heapq.nlargest(20, outDict.keys()):
|
||||||
|
key2 = outDict[key]
|
||||||
|
rec = dbSession.query(db.Recipe).filter(db.Recipe.recipe_id == key2).first()
|
||||||
|
outDict2[key] = (key2, rec.name, rec.url, [r[0] + ": " + r[1] for r in dbSession.query(db.Ingredient.name,
|
||||||
|
db.RecIngred.ingredient_amount).join(db.RecIngred).join(db.Recipe).filter(db.Recipe.recipe_id == key2).all()], rec.img.decode('utf-8'))
|
||||||
|
return outDict2
|
||||||
|
|
||||||
|
def stem(l1):
|
||||||
|
snowball = nltk.SnowballStemmer(language='german')
|
||||||
|
stopset = set(stopwords.words('german'))
|
||||||
|
stopset |= set("(),")
|
||||||
|
l1 = [snowball.stem(l) for l in l1]
|
||||||
|
return l1
|
||||||
|
|
||||||
|
def calcOverlay2(l1, l2):
|
||||||
|
counter = 0
|
||||||
|
for ll in l2.values():
|
||||||
|
for l in ll:
|
||||||
|
if l in l1:
|
||||||
|
counter += 1
|
||||||
|
break
|
||||||
|
|
||||||
|
counter = counter / len(l2)
|
||||||
|
return counter
|
||||||
|
|
||||||
|
# it is assumed that everyone has this
|
||||||
|
defaultArr = ["Wasser", "salz", "pfeffer"]
|
||||||
|
|
||||||
|
|
@ -58,7 +58,7 @@ function renderRecipeList(data){
|
||||||
)
|
)
|
||||||
|
|
||||||
recString = `
|
recString = `
|
||||||
<a href="${data1[2]}">
|
<a href="${data1[2]}" target="_blank">
|
||||||
<div class="card text-white bg-primary mb-3" style="max-width: 100%">
|
<div class="card text-white bg-primary mb-3" style="max-width: 100%">
|
||||||
<div class="card-body recipe-container">
|
<div class="card-body recipe-container">
|
||||||
<div class="row">
|
<div class="row">
|
||||||
Binary file not shown.
|
|
@ -0,0 +1,103 @@
|
||||||
|
import json
|
||||||
|
import cv2
|
||||||
|
import base64
|
||||||
|
import nltk as nltk
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
#import db as db1
|
||||||
|
#import db2 as db2
|
||||||
|
|
||||||
|
def stemWord(word):
|
||||||
|
try:
|
||||||
|
arr = []
|
||||||
|
stopset = set(stopwords.words('german'))
|
||||||
|
stopset |= set("(),")
|
||||||
|
snowball = nltk.SnowballStemmer(language='german')
|
||||||
|
for token in nltk.word_tokenize(word):
|
||||||
|
if token in stopset or len(token) < 4:
|
||||||
|
continue
|
||||||
|
stemmed = snowball.stem(token)
|
||||||
|
arr.append(stemmed)
|
||||||
|
if len(arr) == 0:
|
||||||
|
arr.append("")
|
||||||
|
return arr
|
||||||
|
except:
|
||||||
|
return [""]
|
||||||
|
|
||||||
|
#migrate('./data/recs.json')
|
||||||
|
|
||||||
|
def migrateRecsDb1ToDb2():
|
||||||
|
|
||||||
|
session1 = db1.Session()
|
||||||
|
session2 = db2.Session()
|
||||||
|
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
length = session1.query(db1.Recipe).count()
|
||||||
|
for r1 in list(session1.query(db1.Recipe).all())[int(length/2):]:
|
||||||
|
try:
|
||||||
|
if not bool(session2.query(db2.Recipe).filter(db2.Recipe.name == r1.name).first()):
|
||||||
|
r2 = db2.Recipe(name=r1.name, instructions=r1.instructions, url=r1.url, img=r1.img)
|
||||||
|
|
||||||
|
for ingred in r1.ingredient:
|
||||||
|
ri2 = db2.RecIngred()
|
||||||
|
ingredient2 = session2.query(db2.Ingredient).filter(db2.Ingredient.name == ingred.name).first()
|
||||||
|
if ingredient2 is None:
|
||||||
|
ingredient2 = db2.Ingredient(name=ingred.name)
|
||||||
|
ri2.ingredient_amount = ingred.ingredient_amount
|
||||||
|
ri2.ingredient = ingredient2
|
||||||
|
r2.ingredient.append(ri2)
|
||||||
|
|
||||||
|
session2.add(r2)
|
||||||
|
session2.commit()
|
||||||
|
|
||||||
|
except:
|
||||||
|
session1 = db1.Session()
|
||||||
|
session2 = db2.Session()
|
||||||
|
|
||||||
|
count+=1
|
||||||
|
print(count/length)
|
||||||
|
|
||||||
|
def TrunkDb2():
|
||||||
|
session2 = db2.Session()
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
length = session2.query(db2.Ingredient).count()
|
||||||
|
for i2 in session2.query(db2.Ingredient).all():
|
||||||
|
try:
|
||||||
|
for trunk1 in stem(i2.name):
|
||||||
|
|
||||||
|
ri2 = db2.IngredTrunk()
|
||||||
|
trunk = session2.query(db2.Trunk).filter(db2.Trunk.name == trunk1).first()
|
||||||
|
if trunk is None:
|
||||||
|
trunk = db2.Trunk(name=trunk1)
|
||||||
|
|
||||||
|
if session2.query(db2.IngredTrunk).filter(db2.IngredTrunk.ingredient_name == i2.name, db2.IngredTrunk.trunk_name == trunk1).first() is None:
|
||||||
|
ri2.trunk = trunk
|
||||||
|
i2.trunks.append(ri2)
|
||||||
|
|
||||||
|
session2.commit()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
session2 = db2.Session()
|
||||||
|
|
||||||
|
count+=1
|
||||||
|
print(count/length)
|
||||||
|
|
||||||
|
|
||||||
|
def stem(l1):
|
||||||
|
'''Tokenize and stem word, result is 1d list'''
|
||||||
|
snowball = nltk.SnowballStemmer(language='german')
|
||||||
|
stopset = set(stopwords.words('german'))
|
||||||
|
stopset |= set("(),")
|
||||||
|
l2 = []
|
||||||
|
|
||||||
|
for token in nltk.word_tokenize(l1):
|
||||||
|
token = snowball.stem(token)
|
||||||
|
if token in stopset or not token.isalpha() or len(token) < 2:
|
||||||
|
continue
|
||||||
|
l2.append(token)
|
||||||
|
|
||||||
|
return l2
|
||||||
|
#migrateDb1ToDb2()
|
||||||
|
#TrunkDb2()
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
from application import app
|
||||||
|
from application.search import defaultArr, stem
|
||||||
|
import nltk
|
||||||
|
|
||||||
|
nltk.download('stopwords')
|
||||||
|
nltk.download('punkt')
|
||||||
|
defaultArr = stem(defaultArr)
|
||||||
|
app.run(host="0.0.0.0", port='5001', debug=False, threaded=True)
|
||||||
|
|
||||||
|
|
||||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -1,85 +0,0 @@
|
||||||
import sqlalchemy as db
|
|
||||||
from sqlalchemy import Column, String, Integer, Numeric, Table, DateTime, ARRAY, ForeignKey, create_engine, LargeBinary, Enum, Text
|
|
||||||
from sqlalchemy.orm import sessionmaker, relationship, column_property
|
|
||||||
from datetime import datetime
|
|
||||||
from sqlalchemy.ext.declarative import declarative_base
|
|
||||||
import enum
|
|
||||||
from flask import Flask
|
|
||||||
import time
|
|
||||||
|
|
||||||
engine = db.create_engine('mysql+mysqldb://root@server/fs?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0)
|
|
||||||
|
|
||||||
Base = declarative_base()
|
|
||||||
Session = sessionmaker(bind=engine)
|
|
||||||
|
|
||||||
# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object
|
|
||||||
|
|
||||||
|
|
||||||
class Recipe(Base):
|
|
||||||
__tablename__ = "recipe"
|
|
||||||
recipe_id = Column('recipe_id', Integer, primary_key=True, autoincrement=True)
|
|
||||||
name = Column('name', Text)
|
|
||||||
instructions = Column('instructions', Text)
|
|
||||||
url = Column('url', Text)
|
|
||||||
img = Column('img', LargeBinary(length=(2**32)-1))
|
|
||||||
ingredient = relationship("Ingredient", backref="recipe")
|
|
||||||
trunk = relationship("Trunk", backref="recipe")
|
|
||||||
|
|
||||||
def ingredients(self):
|
|
||||||
l = []
|
|
||||||
for i in self.ingredient:
|
|
||||||
l.append(i.ingredients())
|
|
||||||
return l
|
|
||||||
|
|
||||||
def ingredientDict(self):
|
|
||||||
l = {}
|
|
||||||
for i in self.ingredient:
|
|
||||||
l[i.ingredients()] = [i.ingredient_amount, i.ingredient_amount_mu]
|
|
||||||
return l
|
|
||||||
|
|
||||||
def serialize(self):
|
|
||||||
ingredients = []
|
|
||||||
|
|
||||||
if self.img is not None:
|
|
||||||
img = self.img.decode('utf-8')
|
|
||||||
else:
|
|
||||||
img = None
|
|
||||||
|
|
||||||
data = {
|
|
||||||
"recipe_id": self.recipe_id,
|
|
||||||
"name":self.name,
|
|
||||||
"instructions":self.instructions,
|
|
||||||
"url": self.url,
|
|
||||||
"img": img,
|
|
||||||
"ingredients": self.ingredients()
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
|
|
||||||
class Ingredient(Base):
|
|
||||||
__tablename__ = "ingredient"
|
|
||||||
ingredient_id = Column('ingredient_id', Integer, primary_key=True, autoincrement=True)
|
|
||||||
name = Column('name', Text)
|
|
||||||
ingredient_amount = Column('ingredient_amount', Text)
|
|
||||||
ingredient_amount_mu = Column('ingredient_amount_mu', Text) # measurement unit
|
|
||||||
|
|
||||||
recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'))
|
|
||||||
|
|
||||||
|
|
||||||
class Trunk(Base):
|
|
||||||
__tablename__ = "trunk"
|
|
||||||
trunk_id = Column('trunk_id', Integer, primary_key=True, autoincrement=True)
|
|
||||||
name = Column('name', Text)
|
|
||||||
|
|
||||||
recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'))
|
|
||||||
|
|
||||||
def initDB(counter):
|
|
||||||
try:
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
except Exception as e:
|
|
||||||
print(e)
|
|
||||||
counter += 1
|
|
||||||
if counter < 13:
|
|
||||||
time.sleep(5)
|
|
||||||
initDB(counter)
|
|
||||||
|
|
||||||
initDB(0)
|
|
||||||
BIN
data/images.jpeg
BIN
data/images.jpeg
Binary file not shown.
|
Before Width: | Height: | Size: 10 KiB |
File diff suppressed because one or more lines are too long
|
|
@ -1,27 +0,0 @@
|
||||||
[INFO - 2020-04-20T20:11:22.708Z] GhostDriver - Main - running on port 63975
|
|
||||||
[INFO - 2020-04-20T20:11:24.918Z] Session [1c4df520-8343-11ea-a8a6-19d2d02d716b] - page.settings - {"XSSAuditingEnabled":false,"javascriptCanCloseWindows":true,"javascriptCanOpenWindows":true,"javascriptEnabled":true,"loadImages":true,"localToRemoteUrlAccessEnabled":false,"userAgent":"Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1","webSecurityEnabled":true}
|
|
||||||
[INFO - 2020-04-20T20:11:24.918Z] Session [1c4df520-8343-11ea-a8a6-19d2d02d716b] - page.customHeaders: - {}
|
|
||||||
[INFO - 2020-04-20T20:11:24.918Z] Session [1c4df520-8343-11ea-a8a6-19d2d02d716b] - Session.negotiatedCapabilities - {"browserName":"phantomjs","version":"2.1.1","driverName":"ghostdriver","driverVersion":"1.2.0","platform":"windows-10-32bit","javascriptEnabled":true,"takesScreenshot":true,"handlesAlerts":false,"databaseEnabled":false,"locationContextEnabled":false,"applicationCacheEnabled":false,"browserConnectionEnabled":false,"cssSelectorsEnabled":true,"webStorageEnabled":false,"rotatable":false,"acceptSslCerts":false,"nativeEvents":true,"proxy":{"proxyType":"direct"}}
|
|
||||||
[INFO - 2020-04-20T20:11:24.918Z] SessionManagerReqHand - _postNewSessionCommand - New Session Created: 1c4df520-8343-11ea-a8a6-19d2d02d716b
|
|
||||||
[ERROR - 2020-04-20T20:11:26.927Z] Session [1c4df520-8343-11ea-a8a6-19d2d02d716b] - page.onError - msg: Unhandled promise rejection TypeError: undefined is not an object (evaluating 'w.set')
|
|
||||||
|
|
||||||
phantomjs://platform/console++.js:263 in error
|
|
||||||
[ERROR - 2020-04-20T20:11:26.927Z] Session [1c4df520-8343-11ea-a8a6-19d2d02d716b] - page.onError - stack:
|
|
||||||
(anonymous function) (https://www.chefkoch.de/rezepte/207581086939665/Nannie-s-schnelle-Kuchen.html:8626)
|
|
||||||
|
|
||||||
phantomjs://platform/console++.js:263 in error
|
|
||||||
[ERROR - 2020-04-20T20:11:28.646Z] WebElementLocator - _handleLocateCommand - Element(s) NOT Found: GAVE UP. Search Stop Time: 15874134[INFO - 2020-04-20T20:12:30.427Z] SessionManagerReqHand - [INFO - 2020-04-20T20:13:35.766Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:17:30.429Z] SessionManagerReqHand - [INFO - 2020-04-20T20:18:35.767Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:22:30.429Z] SessionManagerReqHand - [INFO - 2020-04-20T20:23:35.768Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:27:30.429Z] SessionManagerReqHand - [INFO - 2020-04-20T20:28:35.771Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:32:30.439Z] SessionManagerReqHand - [INFO - 2020-04-20T20:33:35.774Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:37:30.439Z] SessionManagerReqHand - [INFO - 2020-04-20T20:38:35.774Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:42:30.439Z] SessionManagerReqHand - [INFO - 2020-04-20T20:43:35.784Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:47:30.439Z] SessionManagerReqHand - [INFO - 2020-04-20T20:48:35.787Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:52:30.439Z] SessionManagerReqHand - [INFO - 2020-04-20T20:53:35.788Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T20:57:30.440Z] SessionManagerReqHand - [INFO - 2020-04-20T20:58:35.788Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:02:30.440Z] SessionManagerReqHand - [INFO - 2020-04-20T21:03:35.789Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:07:30.445Z] SessionManagerReqHand - [INFO - 2020-04-20T21:08:35.789Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:12:30.457Z] SessionManagerReqHand - [INFO - 2020-04-20T21:13:35.788Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:17:30.457Z] SessionManagerReqHand - [INFO - 2020-04-20T21:18:35.798Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:22:30.459Z] SessionManagerReqHand - [INFO - 2020-04-20T21:23:35.803Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:27:30.460Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
OW
|
|
||||||
O - 2020-04-20T21:22:12.188Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:27:12.188Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
ns - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:07:14.184Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:12:14.184Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:17:14.190Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:22:14.192Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:27:14.196Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
242Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:19:25.242Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:24:25.249Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
[INFO - 2020-04-20T21:27:31.799Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
179Z] SessionManagerReqHand - [INFO - 2020-04-20T21:11:22.721Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:13:54.176Z] SessionManagerReqHand - [INFO - 2020-04-20T21:16:22.724Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:18:54.177Z] SessionManagerReqHand - [INFO - 2020-04-20T21:21:22.725Z] SessionManagerReqHand - _cleanupWindowlessSes[INFO - 2020-04-20T21:23:54.181Z] SessionManagerReqHand - [INFO - 2020-04-20T21:26:22.728Z] SessionManagerReqHand - _cleanupWindowlessSessions - Asynchronous Sessions clean-up phase starting NOW
|
|
||||||
52
migrate.py
52
migrate.py
|
|
@ -1,52 +0,0 @@
|
||||||
import json
|
|
||||||
import cv2
|
|
||||||
import base64
|
|
||||||
import nltk as nltk
|
|
||||||
from nltk.corpus import stopwords
|
|
||||||
from application.db import Session, Recipe, Ingredient, Trunk
|
|
||||||
|
|
||||||
def stemWord(word):
|
|
||||||
try:
|
|
||||||
arr = []
|
|
||||||
stopset = set(stopwords.words('german'))
|
|
||||||
stopset |= set("(),")
|
|
||||||
snowball = nltk.SnowballStemmer(language='german')
|
|
||||||
for token in nltk.word_tokenize(word):
|
|
||||||
if token in stopset or len(token) < 4:
|
|
||||||
continue
|
|
||||||
stemmed = snowball.stem(token)
|
|
||||||
arr.append(stemmed)
|
|
||||||
if len(arr) == 0:
|
|
||||||
arr.append("")
|
|
||||||
return arr
|
|
||||||
except:
|
|
||||||
return [""]
|
|
||||||
|
|
||||||
def migrate(path):
|
|
||||||
recs = ""
|
|
||||||
with open(path, encoding="utf-8") as file:
|
|
||||||
recs = json.load(file)
|
|
||||||
|
|
||||||
dbSession = Session()
|
|
||||||
counter = 0
|
|
||||||
leng = len(recs)
|
|
||||||
for key, value in recs.items():
|
|
||||||
name=key
|
|
||||||
resString=value[0]
|
|
||||||
link=value[2]
|
|
||||||
img=value[3].encode()
|
|
||||||
|
|
||||||
r = Recipe(name=name, instructions=resString, url=link, img=img)
|
|
||||||
|
|
||||||
for x, y in value[1].items():
|
|
||||||
a = Ingredient(name=x, ingredient_amount=y)
|
|
||||||
r.ingredient.append(a)
|
|
||||||
for x in stemWord(a.name):
|
|
||||||
t = Trunk(name=x)
|
|
||||||
r.trunk.append(t)
|
|
||||||
dbSession.add(r)
|
|
||||||
dbSession.commit()
|
|
||||||
counter+=1
|
|
||||||
print(counter/leng)
|
|
||||||
|
|
||||||
#migrate('./data/recs.json')
|
|
||||||
11
run.py
11
run.py
|
|
@ -1,11 +0,0 @@
|
||||||
from application import app
|
|
||||||
import nltk
|
|
||||||
from search import defaultArr, stem
|
|
||||||
|
|
||||||
|
|
||||||
nltk.download('stopwords')
|
|
||||||
nltk.download('punkt')
|
|
||||||
delattr = stem(defaultArr)
|
|
||||||
app.run(host="0.0.0.0", port='5001', debug=True, threaded=True)
|
|
||||||
|
|
||||||
|
|
||||||
102
search.py
102
search.py
|
|
@ -1,102 +0,0 @@
|
||||||
|
|
||||||
from application.db import Session, Recipe, Ingredient, Trunk
|
|
||||||
from flask import g
|
|
||||||
import nltk as nltk
|
|
||||||
from nltk.corpus import stopwords
|
|
||||||
import time
|
|
||||||
import heapq
|
|
||||||
from collections import Counter
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def fastes(inputArr):
|
|
||||||
indx = {}
|
|
||||||
dbSession = g.session
|
|
||||||
for inpu in inputArr:
|
|
||||||
ids = []
|
|
||||||
for recipe_id in dbSession.query(Trunk.recipe_id).filter(Trunk.name == inpu).all():
|
|
||||||
if str(recipe_id[0]) not in indx:
|
|
||||||
indx[str(recipe_id[0])] = 0
|
|
||||||
|
|
||||||
indx[str(recipe_id[0])] += 1
|
|
||||||
return(indx)
|
|
||||||
|
|
||||||
def stemInput(inputArr):
|
|
||||||
inputArr2 = []
|
|
||||||
|
|
||||||
snowball = nltk.SnowballStemmer(language='german')
|
|
||||||
stopset = set(stopwords.words('german'))
|
|
||||||
for word in inputArr:
|
|
||||||
if word in stopset:
|
|
||||||
continue
|
|
||||||
inputArr2.append(snowball.stem(word))
|
|
||||||
return inputArr2
|
|
||||||
#
|
|
||||||
|
|
||||||
def getRecDict(indx, inputArr):
|
|
||||||
dbSession = g.session
|
|
||||||
|
|
||||||
outDict = {}
|
|
||||||
k = Counter(indx)
|
|
||||||
# Finding 1000 highest values TODO: this is not correct
|
|
||||||
indx = k.most_common(1000)
|
|
||||||
indx = dict(indx)
|
|
||||||
for key, value in indx.items():
|
|
||||||
ingred = [x[0] for x in dbSession.query(Trunk.name).filter(Trunk.recipe_id==int(key)).all()]
|
|
||||||
outDict[calcOverlay(inputArr, ingred)] = int(key)
|
|
||||||
|
|
||||||
outDict2 = {}
|
|
||||||
for key in heapq.nlargest(10, outDict.keys()):
|
|
||||||
key2 = outDict[key]
|
|
||||||
rec = dbSession.query(Recipe).filter(Recipe.recipe_id==key2).first()
|
|
||||||
outDict2[key] = (key2, rec.name, rec.url, [r[0] + ": " + r[1] for r in dbSession.query(Ingredient.name, Ingredient.ingredient_amount).filter(Ingredient.recipe_id==key2).all()], rec.img.decode('utf-8'))
|
|
||||||
return outDict2
|
|
||||||
|
|
||||||
def printDict(indx, inputArr):
|
|
||||||
outDict = getRecDict(indx, inputArr)
|
|
||||||
for key, value in sorted(outDict.items()):
|
|
||||||
if key >= 0.3:
|
|
||||||
|
|
||||||
print(key, value[0], value[1])
|
|
||||||
for xx in value[2]:
|
|
||||||
print("\t", xx[0])
|
|
||||||
|
|
||||||
|
|
||||||
def stem(l1):
|
|
||||||
snowball = nltk.SnowballStemmer(language='german')
|
|
||||||
stopset = set(stopwords.words('german'))
|
|
||||||
stopset |= set("(),")
|
|
||||||
|
|
||||||
l1 = [snowball.stem(l) for l in l1]
|
|
||||||
return l1
|
|
||||||
|
|
||||||
def calcOverlay(l1, l2):
|
|
||||||
counter = 0
|
|
||||||
for l in l1:
|
|
||||||
if l not in defaultArr:
|
|
||||||
if l in l2:
|
|
||||||
#print(l)
|
|
||||||
counter +=1
|
|
||||||
counter = counter / len(l2)
|
|
||||||
return counter
|
|
||||||
|
|
||||||
|
|
||||||
#inputArr = ["reis", "tofu", "bohnen", "kichererbsen", "hackfleisch"]
|
|
||||||
defaultArr = ["Wasser", "salz", "pfeffer"] # it is assumed that everyone has this
|
|
||||||
#inputArr += defaultArr
|
|
||||||
maxMissing = 10
|
|
||||||
#
|
|
||||||
#stemmed = stemInput(inputArr)
|
|
||||||
#
|
|
||||||
#start = time.time()
|
|
||||||
#indx = faster(stemmed)
|
|
||||||
#end = time.time()
|
|
||||||
#printDict(indx)
|
|
||||||
#print("\n", end - start, "\n")
|
|
||||||
#
|
|
||||||
#
|
|
||||||
#start = time.time()
|
|
||||||
#indx = fastes(stemmed)
|
|
||||||
#end = time.time()
|
|
||||||
#printDict(indx)
|
|
||||||
#print("\n", end - start, "\n")
|
|
||||||
BIN
test.sqlite
BIN
test.sqlite
Binary file not shown.
Loading…
Reference in New Issue