diff --git a/__pycache__/migrate.cpython-37.pyc b/__pycache__/migrate.cpython-37.pyc index baf2651..220a3a2 100644 Binary files a/__pycache__/migrate.cpython-37.pyc and b/__pycache__/migrate.cpython-37.pyc differ diff --git a/application/__pycache__/db2.cpython-37.pyc b/application/__pycache__/db2.cpython-37.pyc new file mode 100644 index 0000000..4e39f1a Binary files /dev/null and b/application/__pycache__/db2.cpython-37.pyc differ diff --git a/migrate.py b/migrate.py deleted file mode 100644 index 1cc221f..0000000 --- a/migrate.py +++ /dev/null @@ -1,52 +0,0 @@ -import json -import cv2 -import base64 -import nltk as nltk -from nltk.corpus import stopwords -from application.db import Session, Recipe, Ingredient, Trunk - -def stemWord(word): - try: - arr = [] - stopset = set(stopwords.words('german')) - stopset |= set("(),") - snowball = nltk.SnowballStemmer(language='german') - for token in nltk.word_tokenize(word): - if token in stopset or len(token) < 4: - continue - stemmed = snowball.stem(token) - arr.append(stemmed) - if len(arr) == 0: - arr.append("") - return arr - except: - return [""] - -def migrate(path): - recs = "" - with open(path, encoding="utf-8") as file: - recs = json.load(file) - - dbSession = Session() - counter = 0 - leng = len(recs) - for key, value in recs.items(): - name=key - resString=value[0] - link=value[2] - img=value[3].encode() - - r = Recipe(name=name, instructions=resString, url=link, img=img) - - for x, y in value[1].items(): - a = Ingredient(name=x, ingredient_amount=y) - r.ingredient.append(a) - for x in stemWord(a.name): - t = Trunk(name=x) - r.trunk.append(t) - dbSession.add(r) - dbSession.commit() - counter+=1 - print(counter/leng) - -#migrate('./data/recs.json') diff --git a/migration/__pycache__/db.cpython-37.pyc b/migration/__pycache__/db.cpython-37.pyc new file mode 100644 index 0000000..7719d0e Binary files /dev/null and b/migration/__pycache__/db.cpython-37.pyc differ diff --git a/migration/__pycache__/db2.cpython-37.pyc b/migration/__pycache__/db2.cpython-37.pyc new file mode 100644 index 0000000..4d999d6 Binary files /dev/null and b/migration/__pycache__/db2.cpython-37.pyc differ diff --git a/migration/db.py b/migration/db.py new file mode 100644 index 0000000..032d5c6 --- /dev/null +++ b/migration/db.py @@ -0,0 +1,56 @@ +import sqlalchemy as db +from sqlalchemy import Column, String, Integer, Numeric, Table, DateTime, ARRAY, ForeignKey, create_engine, LargeBinary, Enum, Text +from sqlalchemy.orm import sessionmaker, relationship, column_property +from datetime import datetime +from sqlalchemy.ext.declarative import declarative_base +import enum +from flask import Flask +import time + +engine = db.create_engine('mysql+mysqldb://root@server/fs?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0) + +Base = declarative_base() +Session = sessionmaker(bind=engine) + +# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object + + +class Recipe(Base): + __tablename__ = "recipe" + recipe_id = Column('recipe_id', Integer, primary_key=True, autoincrement=True) + name = Column('name', Text) + instructions = Column('instructions', Text) + url = Column('url', Text) + img = Column('img', LargeBinary(length=(2**32)-1)) + ingredient = relationship("Ingredient", backref="recipe") + trunk = relationship("Trunk", backref="recipe") + + +class Ingredient(Base): + __tablename__ = "ingredient" + ingredient_id = Column('ingredient_id', Integer, primary_key=True, autoincrement=True) + name = Column('name', Text) + ingredient_amount = Column('ingredient_amount', Text) + ingredient_amount_mu = Column('ingredient_amount_mu', Text) # measurement unit + + recipe_id = Column(Integer, ForeignKey('recipe.recipe_id')) + + +class Trunk(Base): + __tablename__ = "trunk" + trunk_id = Column('trunk_id', Integer, primary_key=True, autoincrement=True) + name = Column('name', Text) + + recipe_id = Column(Integer, ForeignKey('recipe.recipe_id')) + +def initDB(counter): + try: + Base.metadata.create_all(engine) + except Exception as e: + print(e) + counter += 1 + if counter < 13: + time.sleep(5) + initDB(counter) + +initDB(0) \ No newline at end of file diff --git a/migration/db2.py b/migration/db2.py new file mode 100644 index 0000000..52db64e --- /dev/null +++ b/migration/db2.py @@ -0,0 +1,70 @@ +import sqlalchemy as db +from sqlalchemy import Column, String, Integer, Numeric, Table, DateTime, ARRAY, ForeignKey, create_engine, LargeBinary, Enum, Text +from sqlalchemy.orm import sessionmaker, relationship, column_property +from datetime import datetime +from sqlalchemy.ext.declarative import declarative_base +import enum +from flask import Flask +import time + +engine = db.create_engine('mysql+mysqldb://root@server/fs2?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0) + +Base = declarative_base() +Session = sessionmaker(bind=engine, autoflush=False) + +# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object + +class Recipe(Base): + __tablename__ = "recipe" + recipe_id = Column('recipe_id', Integer, primary_key=True, autoincrement=True) + name = Column('name', Text) + instructions = Column('instructions', Text) + url = Column('url', Text) + img = Column('img', LargeBinary(length=(2**32)-1)) + imgURL = Column('imgURL', Text) + ingredient = relationship("RecIngred", back_populates="recipe") + +class RecIngred(Base): + + __tablename__ = 'recingred' + ingredient_amount = Column('ingredient_amount', Text) + ingredient_amount_mu = Column('ingredient_amount_mu', Text) # measurement unit + + recipe = relationship("Recipe", back_populates="ingredient") + ingredient = relationship("Ingredient", back_populates="recipe") + + recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'), primary_key=True) + ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True) + + +class Ingredient(Base): + __tablename__ = "ingredient" + name = Column('name', String(200), primary_key=True) + recipe = relationship("RecIngred", back_populates="ingredient") + trunks = relationship("IngredTrunk", back_populates="ingredient") + +class IngredTrunk(Base): + __tablename__ = 'ingredtrunk' + ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True) + trunk_name = Column(String(50), ForeignKey('trunk.name'), primary_key=True) + + ingredient = relationship("Ingredient", back_populates="trunks") + trunk = relationship("Trunk", back_populates="ingredients") + +class Trunk(Base): + __tablename__ = "trunk" + name = Column('name', String(50), primary_key=True) + ingredients = relationship("IngredTrunk", back_populates="trunk") + + +def initDB(counter=0): + try: + Base.metadata.create_all(engine) + except Exception as e: + print(e) + counter += 1 + if counter < 13: + time.sleep(5) + initDB(counter) + +initDB() \ No newline at end of file diff --git a/migration/migrate.py b/migration/migrate.py new file mode 100644 index 0000000..bd1fc80 --- /dev/null +++ b/migration/migrate.py @@ -0,0 +1,60 @@ +import json +import cv2 +import base64 +import nltk as nltk +from nltk.corpus import stopwords +import db as db1 +import db2 as db2 + +def stemWord(word): + try: + arr = [] + stopset = set(stopwords.words('german')) + stopset |= set("(),") + snowball = nltk.SnowballStemmer(language='german') + for token in nltk.word_tokenize(word): + if token in stopset or len(token) < 4: + continue + stemmed = snowball.stem(token) + arr.append(stemmed) + if len(arr) == 0: + arr.append("") + return arr + except: + return [""] + +#migrate('./data/recs.json') + +def migrateDb1ToDb2(): + + session1 = db1.Session() + session2 = db2.Session() + + + count = 0 + length = session1.query(db1.Recipe).count() + for r1 in list(session1.query(db1.Recipe).all())[int(length/2):]: + try: + if not bool(session2.query(db2.Recipe).filter(db2.Recipe.name == r1.name).first()): + r2 = db2.Recipe(name=r1.name, instructions=r1.instructions, url=r1.url, img=r1.img) + + for ingred in r1.ingredient: + ri2 = db2.RecIngred() + ingredient2 = session2.query(db2.Ingredient).filter(db2.Ingredient.name == ingred.name).first() + if ingredient2 is None: + ingredient2 = db2.Ingredient(name=ingred.name) + ri2.ingredient_amount = ingred.ingredient_amount + ri2.ingredient = ingredient2 + r2.ingredient.append(ri2) + + session2.add(r2) + session2.commit() + + except: + session1 = db1.Session() + session2 = db2.Session() + + count+=1 + print(count/length) + +migrateDb1ToDb2() \ No newline at end of file