added migartion

migrate.py

@@ -1,52 +0,0 @@
-import json
-import cv2
-import base64
-import nltk as nltk
-from nltk.corpus import stopwords
-from application.db import Session, Recipe, Ingredient, Trunk
-
-def stemWord(word):
-    try:
-        arr = []
-        stopset = set(stopwords.words('german'))
-        stopset |= set("(),")
-        snowball = nltk.SnowballStemmer(language='german')
-        for token in nltk.word_tokenize(word): 
-            if token in stopset or len(token) < 4:
-                continue
-            stemmed = snowball.stem(token)
-            arr.append(stemmed)
-        if len(arr) == 0:
-            arr.append("")
-        return arr
-    except:
-        return [""]
-
-def migrate(path):
-    recs = ""
-    with open(path, encoding="utf-8") as file:
-        recs = json.load(file)
-    
-    dbSession = Session()
-    counter = 0
-    leng = len(recs)
-    for key, value in recs.items():
-        name=key
-        resString=value[0]
-        link=value[2]
-        img=value[3].encode()
-
-        r = Recipe(name=name, instructions=resString, url=link, img=img)
-
-        for x, y in value[1].items():
-            a = Ingredient(name=x, ingredient_amount=y)
-            r.ingredient.append(a)
-            for x in stemWord(a.name):
-                t = Trunk(name=x)
-                r.trunk.append(t)
-        dbSession.add(r)
-        dbSession.commit()
-        counter+=1
-        print(counter/leng)
-
-#migrate('./data/recs.json')

migration/db.py

@@ -0,0 +1,56 @@
+import sqlalchemy as db
+from sqlalchemy import Column, String, Integer, Numeric, Table, DateTime, ARRAY, ForeignKey, create_engine, LargeBinary, Enum, Text
+from sqlalchemy.orm import sessionmaker, relationship, column_property
+from datetime import datetime
+from sqlalchemy.ext.declarative import declarative_base
+import enum
+from flask import Flask
+import time
+
+engine = db.create_engine('mysql+mysqldb://root@server/fs?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0)
+
+Base = declarative_base()
+Session = sessionmaker(bind=engine)
+
+# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object
+
+
+class Recipe(Base):
+    __tablename__ = "recipe"
+    recipe_id = Column('recipe_id', Integer, primary_key=True, autoincrement=True)
+    name = Column('name', Text)
+    instructions = Column('instructions', Text)
+    url = Column('url', Text)
+    img = Column('img', LargeBinary(length=(2**32)-1))
+    ingredient = relationship("Ingredient", backref="recipe")
+    trunk = relationship("Trunk", backref="recipe")
+
+
+class Ingredient(Base):
+    __tablename__ = "ingredient"
+    ingredient_id = Column('ingredient_id', Integer,  primary_key=True, autoincrement=True)
+    name = Column('name', Text)
+    ingredient_amount = Column('ingredient_amount', Text)
+    ingredient_amount_mu = Column('ingredient_amount_mu', Text)    # measurement unit
+
+    recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'))
+    
+
+class Trunk(Base):
+    __tablename__ = "trunk"
+    trunk_id = Column('trunk_id', Integer,  primary_key=True, autoincrement=True)
+    name = Column('name', Text)
+
+    recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'))
+
+def initDB(counter):
+    try:
+        Base.metadata.create_all(engine)
+    except Exception as e:
+        print(e)
+        counter += 1
+        if counter < 13:
+            time.sleep(5)
+            initDB(counter)
+
+initDB(0)

migration/db2.py

@@ -0,0 +1,70 @@
+import sqlalchemy as db
+from sqlalchemy import Column, String, Integer, Numeric, Table, DateTime, ARRAY, ForeignKey, create_engine, LargeBinary, Enum, Text
+from sqlalchemy.orm import sessionmaker, relationship, column_property
+from datetime import datetime
+from sqlalchemy.ext.declarative import declarative_base
+import enum
+from flask import Flask
+import time
+
+engine = db.create_engine('mysql+mysqldb://root@server/fs2?charset=utf8mb4', echo=False, encoding="utf8", pool_size=1000, max_overflow=0)
+
+Base = declarative_base()
+Session = sessionmaker(bind=engine, autoflush=False)
+
+# https://docs.sqlalchemy.org/en/13/orm/basic_relationships.html#association-object
+
+class Recipe(Base):
+    __tablename__ = "recipe"
+    recipe_id = Column('recipe_id', Integer, primary_key=True, autoincrement=True)
+    name = Column('name', Text)
+    instructions = Column('instructions', Text)
+    url = Column('url', Text)
+    img = Column('img', LargeBinary(length=(2**32)-1))
+    imgURL = Column('imgURL', Text)
+    ingredient = relationship("RecIngred", back_populates="recipe")
+
+class RecIngred(Base):
+
+    __tablename__ = 'recingred'
+    ingredient_amount = Column('ingredient_amount', Text)
+    ingredient_amount_mu = Column('ingredient_amount_mu', Text)    # measurement unit
+
+    recipe = relationship("Recipe", back_populates="ingredient")
+    ingredient = relationship("Ingredient", back_populates="recipe")
+
+    recipe_id = Column(Integer, ForeignKey('recipe.recipe_id'), primary_key=True)
+    ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True)
+
+
+class Ingredient(Base):
+    __tablename__ = "ingredient"
+    name = Column('name', String(200), primary_key=True)
+    recipe = relationship("RecIngred", back_populates="ingredient")
+    trunks = relationship("IngredTrunk", back_populates="ingredient")
+
+class IngredTrunk(Base):
+    __tablename__ = 'ingredtrunk'
+    ingredient_name = Column(String(200), ForeignKey('ingredient.name'), primary_key=True)
+    trunk_name = Column(String(50), ForeignKey('trunk.name'), primary_key=True)
+
+    ingredient = relationship("Ingredient", back_populates="trunks")
+    trunk = relationship("Trunk", back_populates="ingredients")
+
+class Trunk(Base):
+    __tablename__ = "trunk"
+    name = Column('name', String(50), primary_key=True)
+    ingredients = relationship("IngredTrunk", back_populates="trunk")
+
+
+def initDB(counter=0):
+    try:
+        Base.metadata.create_all(engine)
+    except Exception as e:
+        print(e)
+        counter += 1
+        if counter < 13:
+            time.sleep(5)
+            initDB(counter)
+
+initDB()

migration/migrate.py

@@ -0,0 +1,60 @@
+import json
+import cv2
+import base64
+import nltk as nltk
+from nltk.corpus import stopwords
+import db as db1
+import db2 as db2
+
+def stemWord(word):
+    try:
+        arr = []
+        stopset = set(stopwords.words('german'))
+        stopset |= set("(),")
+        snowball = nltk.SnowballStemmer(language='german')
+        for token in nltk.word_tokenize(word): 
+            if token in stopset or len(token) < 4:
+                continue
+            stemmed = snowball.stem(token)
+            arr.append(stemmed)
+        if len(arr) == 0:
+            arr.append("")
+        return arr
+    except:
+        return [""]
+
+#migrate('./data/recs.json')
+
+def migrateDb1ToDb2():
+
+    session1 = db1.Session()
+    session2 = db2.Session()
+ 
+
+    count = 0
+    length = session1.query(db1.Recipe).count()
+    for r1 in list(session1.query(db1.Recipe).all())[int(length/2):]:
+        try:
+            if not bool(session2.query(db2.Recipe).filter(db2.Recipe.name == r1.name).first()):
+                r2 = db2.Recipe(name=r1.name, instructions=r1.instructions, url=r1.url, img=r1.img)
+                
+                for ingred in r1.ingredient:
+                    ri2 = db2.RecIngred()
+                    ingredient2 = session2.query(db2.Ingredient).filter(db2.Ingredient.name == ingred.name).first()
+                    if ingredient2 is None:
+                        ingredient2 = db2.Ingredient(name=ingred.name)
+                    ri2.ingredient_amount = ingred.ingredient_amount
+                    ri2.ingredient = ingredient2
+                    r2.ingredient.append(ri2)
+                
+                session2.add(r2)
+                session2.commit()
+            
+        except:
+            session1 = db1.Session()
+            session2 = db2.Session()
+
+        count+=1
+        print(count/length)
+
+migrateDb1ToDb2()