From 401d3bd8bab82d632acf9fcd84015e7926c23797 Mon Sep 17 00:00:00 2001 From: Patrice Date: Mon, 22 Apr 2019 17:06:20 +0200 Subject: [PATCH] added core func for Golem --- .gitignore => f-ask/.gitignore | 0 {.vscode => f-ask/.vscode}/launch.json | 0 LICENSE.txt => f-ask/LICENSE.txt | 0 MANIFEST.in => f-ask/MANIFEST.in | 0 README.md => f-ask/README.md | 0 README.rst => f-ask/README.rst | 0 {docs => f-ask/docs}/Makefile | 0 {docs => f-ask/docs}/_static/logo-full.png | Bin {docs => f-ask/docs}/_static/logo-sm.png | Bin {docs => f-ask/docs}/_templates/links.html | 0 .../docs}/_templates/sidebarlogo.html | 0 .../docs}/_templates/stayinformed.html | 0 {docs => f-ask/docs}/_themes/LICENSE | 0 {docs => f-ask/docs}/_themes/README | 0 .../docs}/_themes/flask/layout.html | 0 .../docs}/_themes/flask/relations.html | 0 .../docs}/_themes/flask/static/flasky.css_t | 0 {docs => f-ask/docs}/_themes/flask/theme.conf | 0 .../docs}/_themes/flask_theme_support.py | 0 {docs => f-ask/docs}/conf.py | 0 {docs => f-ask/docs}/configuration.rst | 0 {docs => f-ask/docs}/contents.rst.inc | 0 {docs => f-ask/docs}/flaskdocext.py | 0 {docs => f-ask/docs}/getting_started.rst | 0 {docs => f-ask/docs}/index.rst | 0 {docs => f-ask/docs}/make.bat | 0 {docs => f-ask/docs}/requests.rst | 0 {docs => f-ask/docs}/responses.rst | 0 {docs => f-ask/docs}/user_contributions.rst | 0 {flask_ask => f-ask/flask_ask}/__init__.py | 0 {flask_ask => f-ask/flask_ask}/cache.py | 0 {flask_ask => f-ask/flask_ask}/convert.py | 0 {flask_ask => f-ask/flask_ask}/core.py | 0 {flask_ask => f-ask/flask_ask}/models.py | 0 {flask_ask => f-ask/flask_ask}/verifier.py | 0 ngrok.exe => f-ask/ngrok.exe | Bin .../requirements-dev.txt | 0 requirements.txt => f-ask/requirements.txt | 0 .../samples}/audio/playlist_demo/playlist.py | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/audio/simple_demo/ask_audio.py | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/blueprint_demo/demo.py | 0 .../samples}/blueprint_demo/helloworld.py | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/blueprint_demo/templates.yaml | 0 .../samples}/helloworld/helloworld.py | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/historybuff/historybuff.py | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/purchase/IntentSchema.json | 0 {samples => f-ask/samples}/purchase/model.py | 0 .../samples}/purchase/purchase.py | 0 .../samples}/purchase/templates.yaml | 0 {samples => f-ask/samples}/reddit/main.py | 0 .../reddit/speech_assets/IntentSchema.json | 0 .../reddit/speech_assets/SampleUtterances.txt | 0 .../customSlotTypes/NAME_OF_BODY_PARTS | 0 {samples => f-ask/samples}/reddit/test.py | 0 .../samples}/reddit/workouts.yaml | 0 {samples => f-ask/samples}/session/session.py | 0 .../session/speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../customSlotTypes/LIST_OF_COLORS | 0 .../samples}/session/templates.yaml | 0 .../samples}/spacegeek/spacegeek.py | 0 .../spacegeek/speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../samples}/spacegeek/templates.yaml | 0 .../speech_assets/IntentSchema.json | 0 .../speech_assets/SampleUtterances.txt | 0 .../customSlotTypes/LIST_OF_CITIES | 0 .../customSlotTypes/LIST_OF_STATES | 0 .../samples}/tidepooler/templates.yaml | 0 .../samples}/tidepooler/tidepooler.py | 0 setup.cfg => f-ask/setup.cfg | 0 setup.py => f-ask/setup.py | 0 {tests => f-ask/tests}/__init__.py | 0 {tests => f-ask/tests}/test_audio.py | 0 {tests => f-ask/tests}/test_cache.py | 0 {tests => f-ask/tests}/test_core.py | 0 {tests => f-ask/tests}/test_integration.py | 0 ...t_integration_support_entity_resolution.py | 0 {tests => f-ask/tests}/test_samples.py | 0 {tests => f-ask/tests}/test_unicode.py | 0 tox.ini => f-ask/tox.ini | 0 reader/main.py | 0 reader/site.py | 61 ++++++++ reader/tests.py | 10 ++ reader/urlchecker.py | 135 ++++++++++++++++++ 95 files changed, 206 insertions(+) rename .gitignore => f-ask/.gitignore (100%) rename {.vscode => f-ask/.vscode}/launch.json (100%) rename LICENSE.txt => f-ask/LICENSE.txt (100%) rename MANIFEST.in => f-ask/MANIFEST.in (100%) rename README.md => f-ask/README.md (100%) rename README.rst => f-ask/README.rst (100%) rename {docs => f-ask/docs}/Makefile (100%) rename {docs => f-ask/docs}/_static/logo-full.png (100%) rename {docs => f-ask/docs}/_static/logo-sm.png (100%) rename {docs => f-ask/docs}/_templates/links.html (100%) rename {docs => f-ask/docs}/_templates/sidebarlogo.html (100%) rename {docs => f-ask/docs}/_templates/stayinformed.html (100%) rename {docs => f-ask/docs}/_themes/LICENSE (100%) rename {docs => f-ask/docs}/_themes/README (100%) rename {docs => f-ask/docs}/_themes/flask/layout.html (100%) rename {docs => f-ask/docs}/_themes/flask/relations.html (100%) rename {docs => f-ask/docs}/_themes/flask/static/flasky.css_t (100%) rename {docs => f-ask/docs}/_themes/flask/theme.conf (100%) rename {docs => f-ask/docs}/_themes/flask_theme_support.py (100%) rename {docs => f-ask/docs}/conf.py (100%) rename {docs => f-ask/docs}/configuration.rst (100%) rename {docs => f-ask/docs}/contents.rst.inc (100%) rename {docs => f-ask/docs}/flaskdocext.py (100%) rename {docs => f-ask/docs}/getting_started.rst (100%) rename {docs => f-ask/docs}/index.rst (100%) rename {docs => f-ask/docs}/make.bat (100%) rename {docs => f-ask/docs}/requests.rst (100%) rename {docs => f-ask/docs}/responses.rst (100%) rename {docs => f-ask/docs}/user_contributions.rst (100%) rename {flask_ask => f-ask/flask_ask}/__init__.py (100%) rename {flask_ask => f-ask/flask_ask}/cache.py (100%) rename {flask_ask => f-ask/flask_ask}/convert.py (100%) rename {flask_ask => f-ask/flask_ask}/core.py (100%) rename {flask_ask => f-ask/flask_ask}/models.py (100%) rename {flask_ask => f-ask/flask_ask}/verifier.py (100%) rename ngrok.exe => f-ask/ngrok.exe (100%) rename requirements-dev.txt => f-ask/requirements-dev.txt (100%) rename requirements.txt => f-ask/requirements.txt (100%) rename {samples => f-ask/samples}/audio/playlist_demo/playlist.py (100%) rename {samples => f-ask/samples}/audio/playlist_demo/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/audio/playlist_demo/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/audio/simple_demo/ask_audio.py (100%) rename {samples => f-ask/samples}/audio/simple_demo/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/audio/simple_demo/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/blueprint_demo/demo.py (100%) rename {samples => f-ask/samples}/blueprint_demo/helloworld.py (100%) rename {samples => f-ask/samples}/blueprint_demo/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/blueprint_demo/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/blueprint_demo/templates.yaml (100%) rename {samples => f-ask/samples}/helloworld/helloworld.py (100%) rename {samples => f-ask/samples}/helloworld/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/helloworld/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/historybuff/historybuff.py (100%) rename {samples => f-ask/samples}/historybuff/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/historybuff/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/purchase/IntentSchema.json (100%) rename {samples => f-ask/samples}/purchase/model.py (100%) rename {samples => f-ask/samples}/purchase/purchase.py (100%) rename {samples => f-ask/samples}/purchase/templates.yaml (100%) rename {samples => f-ask/samples}/reddit/main.py (100%) rename {samples => f-ask/samples}/reddit/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/reddit/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/reddit/speech_assets/customSlotTypes/NAME_OF_BODY_PARTS (100%) rename {samples => f-ask/samples}/reddit/test.py (100%) rename {samples => f-ask/samples}/reddit/workouts.yaml (100%) rename {samples => f-ask/samples}/session/session.py (100%) rename {samples => f-ask/samples}/session/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/session/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/session/speech_assets/customSlotTypes/LIST_OF_COLORS (100%) rename {samples => f-ask/samples}/session/templates.yaml (100%) rename {samples => f-ask/samples}/spacegeek/spacegeek.py (100%) rename {samples => f-ask/samples}/spacegeek/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/spacegeek/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/spacegeek/templates.yaml (100%) rename {samples => f-ask/samples}/tidepooler/speech_assets/IntentSchema.json (100%) rename {samples => f-ask/samples}/tidepooler/speech_assets/SampleUtterances.txt (100%) rename {samples => f-ask/samples}/tidepooler/speech_assets/customSlotTypes/LIST_OF_CITIES (100%) rename {samples => f-ask/samples}/tidepooler/speech_assets/customSlotTypes/LIST_OF_STATES (100%) rename {samples => f-ask/samples}/tidepooler/templates.yaml (100%) rename {samples => f-ask/samples}/tidepooler/tidepooler.py (100%) rename setup.cfg => f-ask/setup.cfg (100%) rename setup.py => f-ask/setup.py (100%) rename {tests => f-ask/tests}/__init__.py (100%) rename {tests => f-ask/tests}/test_audio.py (100%) rename {tests => f-ask/tests}/test_cache.py (100%) rename {tests => f-ask/tests}/test_core.py (100%) rename {tests => f-ask/tests}/test_integration.py (100%) rename {tests => f-ask/tests}/test_integration_support_entity_resolution.py (100%) rename {tests => f-ask/tests}/test_samples.py (100%) rename {tests => f-ask/tests}/test_unicode.py (100%) rename tox.ini => f-ask/tox.ini (100%) create mode 100644 reader/main.py create mode 100644 reader/site.py create mode 100644 reader/tests.py create mode 100644 reader/urlchecker.py diff --git a/.gitignore b/f-ask/.gitignore similarity index 100% rename from .gitignore rename to f-ask/.gitignore diff --git a/.vscode/launch.json b/f-ask/.vscode/launch.json similarity index 100% rename from .vscode/launch.json rename to f-ask/.vscode/launch.json diff --git a/LICENSE.txt b/f-ask/LICENSE.txt similarity index 100% rename from LICENSE.txt rename to f-ask/LICENSE.txt diff --git a/MANIFEST.in b/f-ask/MANIFEST.in similarity index 100% rename from MANIFEST.in rename to f-ask/MANIFEST.in diff --git a/README.md b/f-ask/README.md similarity index 100% rename from README.md rename to f-ask/README.md diff --git a/README.rst b/f-ask/README.rst similarity index 100% rename from README.rst rename to f-ask/README.rst diff --git a/docs/Makefile b/f-ask/docs/Makefile similarity index 100% rename from docs/Makefile rename to f-ask/docs/Makefile diff --git a/docs/_static/logo-full.png b/f-ask/docs/_static/logo-full.png similarity index 100% rename from docs/_static/logo-full.png rename to f-ask/docs/_static/logo-full.png diff --git a/docs/_static/logo-sm.png b/f-ask/docs/_static/logo-sm.png similarity index 100% rename from docs/_static/logo-sm.png rename to f-ask/docs/_static/logo-sm.png diff --git a/docs/_templates/links.html b/f-ask/docs/_templates/links.html similarity index 100% rename from docs/_templates/links.html rename to f-ask/docs/_templates/links.html diff --git a/docs/_templates/sidebarlogo.html b/f-ask/docs/_templates/sidebarlogo.html similarity index 100% rename from docs/_templates/sidebarlogo.html rename to f-ask/docs/_templates/sidebarlogo.html diff --git a/docs/_templates/stayinformed.html b/f-ask/docs/_templates/stayinformed.html similarity index 100% rename from docs/_templates/stayinformed.html rename to f-ask/docs/_templates/stayinformed.html diff --git a/docs/_themes/LICENSE b/f-ask/docs/_themes/LICENSE similarity index 100% rename from docs/_themes/LICENSE rename to f-ask/docs/_themes/LICENSE diff --git a/docs/_themes/README b/f-ask/docs/_themes/README similarity index 100% rename from docs/_themes/README rename to f-ask/docs/_themes/README diff --git a/docs/_themes/flask/layout.html b/f-ask/docs/_themes/flask/layout.html similarity index 100% rename from docs/_themes/flask/layout.html rename to f-ask/docs/_themes/flask/layout.html diff --git a/docs/_themes/flask/relations.html b/f-ask/docs/_themes/flask/relations.html similarity index 100% rename from docs/_themes/flask/relations.html rename to f-ask/docs/_themes/flask/relations.html diff --git a/docs/_themes/flask/static/flasky.css_t b/f-ask/docs/_themes/flask/static/flasky.css_t similarity index 100% rename from docs/_themes/flask/static/flasky.css_t rename to f-ask/docs/_themes/flask/static/flasky.css_t diff --git a/docs/_themes/flask/theme.conf b/f-ask/docs/_themes/flask/theme.conf similarity index 100% rename from docs/_themes/flask/theme.conf rename to f-ask/docs/_themes/flask/theme.conf diff --git a/docs/_themes/flask_theme_support.py b/f-ask/docs/_themes/flask_theme_support.py similarity index 100% rename from docs/_themes/flask_theme_support.py rename to f-ask/docs/_themes/flask_theme_support.py diff --git a/docs/conf.py b/f-ask/docs/conf.py similarity index 100% rename from docs/conf.py rename to f-ask/docs/conf.py diff --git a/docs/configuration.rst b/f-ask/docs/configuration.rst similarity index 100% rename from docs/configuration.rst rename to f-ask/docs/configuration.rst diff --git a/docs/contents.rst.inc b/f-ask/docs/contents.rst.inc similarity index 100% rename from docs/contents.rst.inc rename to f-ask/docs/contents.rst.inc diff --git a/docs/flaskdocext.py b/f-ask/docs/flaskdocext.py similarity index 100% rename from docs/flaskdocext.py rename to f-ask/docs/flaskdocext.py diff --git a/docs/getting_started.rst b/f-ask/docs/getting_started.rst similarity index 100% rename from docs/getting_started.rst rename to f-ask/docs/getting_started.rst diff --git a/docs/index.rst b/f-ask/docs/index.rst similarity index 100% rename from docs/index.rst rename to f-ask/docs/index.rst diff --git a/docs/make.bat b/f-ask/docs/make.bat similarity index 100% rename from docs/make.bat rename to f-ask/docs/make.bat diff --git a/docs/requests.rst b/f-ask/docs/requests.rst similarity index 100% rename from docs/requests.rst rename to f-ask/docs/requests.rst diff --git a/docs/responses.rst b/f-ask/docs/responses.rst similarity index 100% rename from docs/responses.rst rename to f-ask/docs/responses.rst diff --git a/docs/user_contributions.rst b/f-ask/docs/user_contributions.rst similarity index 100% rename from docs/user_contributions.rst rename to f-ask/docs/user_contributions.rst diff --git a/flask_ask/__init__.py b/f-ask/flask_ask/__init__.py similarity index 100% rename from flask_ask/__init__.py rename to f-ask/flask_ask/__init__.py diff --git a/flask_ask/cache.py b/f-ask/flask_ask/cache.py similarity index 100% rename from flask_ask/cache.py rename to f-ask/flask_ask/cache.py diff --git a/flask_ask/convert.py b/f-ask/flask_ask/convert.py similarity index 100% rename from flask_ask/convert.py rename to f-ask/flask_ask/convert.py diff --git a/flask_ask/core.py b/f-ask/flask_ask/core.py similarity index 100% rename from flask_ask/core.py rename to f-ask/flask_ask/core.py diff --git a/flask_ask/models.py b/f-ask/flask_ask/models.py similarity index 100% rename from flask_ask/models.py rename to f-ask/flask_ask/models.py diff --git a/flask_ask/verifier.py b/f-ask/flask_ask/verifier.py similarity index 100% rename from flask_ask/verifier.py rename to f-ask/flask_ask/verifier.py diff --git a/ngrok.exe b/f-ask/ngrok.exe similarity index 100% rename from ngrok.exe rename to f-ask/ngrok.exe diff --git a/requirements-dev.txt b/f-ask/requirements-dev.txt similarity index 100% rename from requirements-dev.txt rename to f-ask/requirements-dev.txt diff --git a/requirements.txt b/f-ask/requirements.txt similarity index 100% rename from requirements.txt rename to f-ask/requirements.txt diff --git a/samples/audio/playlist_demo/playlist.py b/f-ask/samples/audio/playlist_demo/playlist.py similarity index 100% rename from samples/audio/playlist_demo/playlist.py rename to f-ask/samples/audio/playlist_demo/playlist.py diff --git a/samples/audio/playlist_demo/speech_assets/IntentSchema.json b/f-ask/samples/audio/playlist_demo/speech_assets/IntentSchema.json similarity index 100% rename from samples/audio/playlist_demo/speech_assets/IntentSchema.json rename to f-ask/samples/audio/playlist_demo/speech_assets/IntentSchema.json diff --git a/samples/audio/playlist_demo/speech_assets/SampleUtterances.txt b/f-ask/samples/audio/playlist_demo/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/audio/playlist_demo/speech_assets/SampleUtterances.txt rename to f-ask/samples/audio/playlist_demo/speech_assets/SampleUtterances.txt diff --git a/samples/audio/simple_demo/ask_audio.py b/f-ask/samples/audio/simple_demo/ask_audio.py similarity index 100% rename from samples/audio/simple_demo/ask_audio.py rename to f-ask/samples/audio/simple_demo/ask_audio.py diff --git a/samples/audio/simple_demo/speech_assets/IntentSchema.json b/f-ask/samples/audio/simple_demo/speech_assets/IntentSchema.json similarity index 100% rename from samples/audio/simple_demo/speech_assets/IntentSchema.json rename to f-ask/samples/audio/simple_demo/speech_assets/IntentSchema.json diff --git a/samples/audio/simple_demo/speech_assets/SampleUtterances.txt b/f-ask/samples/audio/simple_demo/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/audio/simple_demo/speech_assets/SampleUtterances.txt rename to f-ask/samples/audio/simple_demo/speech_assets/SampleUtterances.txt diff --git a/samples/blueprint_demo/demo.py b/f-ask/samples/blueprint_demo/demo.py similarity index 100% rename from samples/blueprint_demo/demo.py rename to f-ask/samples/blueprint_demo/demo.py diff --git a/samples/blueprint_demo/helloworld.py b/f-ask/samples/blueprint_demo/helloworld.py similarity index 100% rename from samples/blueprint_demo/helloworld.py rename to f-ask/samples/blueprint_demo/helloworld.py diff --git a/samples/blueprint_demo/speech_assets/IntentSchema.json b/f-ask/samples/blueprint_demo/speech_assets/IntentSchema.json similarity index 100% rename from samples/blueprint_demo/speech_assets/IntentSchema.json rename to f-ask/samples/blueprint_demo/speech_assets/IntentSchema.json diff --git a/samples/blueprint_demo/speech_assets/SampleUtterances.txt b/f-ask/samples/blueprint_demo/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/blueprint_demo/speech_assets/SampleUtterances.txt rename to f-ask/samples/blueprint_demo/speech_assets/SampleUtterances.txt diff --git a/samples/blueprint_demo/templates.yaml b/f-ask/samples/blueprint_demo/templates.yaml similarity index 100% rename from samples/blueprint_demo/templates.yaml rename to f-ask/samples/blueprint_demo/templates.yaml diff --git a/samples/helloworld/helloworld.py b/f-ask/samples/helloworld/helloworld.py similarity index 100% rename from samples/helloworld/helloworld.py rename to f-ask/samples/helloworld/helloworld.py diff --git a/samples/helloworld/speech_assets/IntentSchema.json b/f-ask/samples/helloworld/speech_assets/IntentSchema.json similarity index 100% rename from samples/helloworld/speech_assets/IntentSchema.json rename to f-ask/samples/helloworld/speech_assets/IntentSchema.json diff --git a/samples/helloworld/speech_assets/SampleUtterances.txt b/f-ask/samples/helloworld/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/helloworld/speech_assets/SampleUtterances.txt rename to f-ask/samples/helloworld/speech_assets/SampleUtterances.txt diff --git a/samples/historybuff/historybuff.py b/f-ask/samples/historybuff/historybuff.py similarity index 100% rename from samples/historybuff/historybuff.py rename to f-ask/samples/historybuff/historybuff.py diff --git a/samples/historybuff/speech_assets/IntentSchema.json b/f-ask/samples/historybuff/speech_assets/IntentSchema.json similarity index 100% rename from samples/historybuff/speech_assets/IntentSchema.json rename to f-ask/samples/historybuff/speech_assets/IntentSchema.json diff --git a/samples/historybuff/speech_assets/SampleUtterances.txt b/f-ask/samples/historybuff/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/historybuff/speech_assets/SampleUtterances.txt rename to f-ask/samples/historybuff/speech_assets/SampleUtterances.txt diff --git a/samples/purchase/IntentSchema.json b/f-ask/samples/purchase/IntentSchema.json similarity index 100% rename from samples/purchase/IntentSchema.json rename to f-ask/samples/purchase/IntentSchema.json diff --git a/samples/purchase/model.py b/f-ask/samples/purchase/model.py similarity index 100% rename from samples/purchase/model.py rename to f-ask/samples/purchase/model.py diff --git a/samples/purchase/purchase.py b/f-ask/samples/purchase/purchase.py similarity index 100% rename from samples/purchase/purchase.py rename to f-ask/samples/purchase/purchase.py diff --git a/samples/purchase/templates.yaml b/f-ask/samples/purchase/templates.yaml similarity index 100% rename from samples/purchase/templates.yaml rename to f-ask/samples/purchase/templates.yaml diff --git a/samples/reddit/main.py b/f-ask/samples/reddit/main.py similarity index 100% rename from samples/reddit/main.py rename to f-ask/samples/reddit/main.py diff --git a/samples/reddit/speech_assets/IntentSchema.json b/f-ask/samples/reddit/speech_assets/IntentSchema.json similarity index 100% rename from samples/reddit/speech_assets/IntentSchema.json rename to f-ask/samples/reddit/speech_assets/IntentSchema.json diff --git a/samples/reddit/speech_assets/SampleUtterances.txt b/f-ask/samples/reddit/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/reddit/speech_assets/SampleUtterances.txt rename to f-ask/samples/reddit/speech_assets/SampleUtterances.txt diff --git a/samples/reddit/speech_assets/customSlotTypes/NAME_OF_BODY_PARTS b/f-ask/samples/reddit/speech_assets/customSlotTypes/NAME_OF_BODY_PARTS similarity index 100% rename from samples/reddit/speech_assets/customSlotTypes/NAME_OF_BODY_PARTS rename to f-ask/samples/reddit/speech_assets/customSlotTypes/NAME_OF_BODY_PARTS diff --git a/samples/reddit/test.py b/f-ask/samples/reddit/test.py similarity index 100% rename from samples/reddit/test.py rename to f-ask/samples/reddit/test.py diff --git a/samples/reddit/workouts.yaml b/f-ask/samples/reddit/workouts.yaml similarity index 100% rename from samples/reddit/workouts.yaml rename to f-ask/samples/reddit/workouts.yaml diff --git a/samples/session/session.py b/f-ask/samples/session/session.py similarity index 100% rename from samples/session/session.py rename to f-ask/samples/session/session.py diff --git a/samples/session/speech_assets/IntentSchema.json b/f-ask/samples/session/speech_assets/IntentSchema.json similarity index 100% rename from samples/session/speech_assets/IntentSchema.json rename to f-ask/samples/session/speech_assets/IntentSchema.json diff --git a/samples/session/speech_assets/SampleUtterances.txt b/f-ask/samples/session/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/session/speech_assets/SampleUtterances.txt rename to f-ask/samples/session/speech_assets/SampleUtterances.txt diff --git a/samples/session/speech_assets/customSlotTypes/LIST_OF_COLORS b/f-ask/samples/session/speech_assets/customSlotTypes/LIST_OF_COLORS similarity index 100% rename from samples/session/speech_assets/customSlotTypes/LIST_OF_COLORS rename to f-ask/samples/session/speech_assets/customSlotTypes/LIST_OF_COLORS diff --git a/samples/session/templates.yaml b/f-ask/samples/session/templates.yaml similarity index 100% rename from samples/session/templates.yaml rename to f-ask/samples/session/templates.yaml diff --git a/samples/spacegeek/spacegeek.py b/f-ask/samples/spacegeek/spacegeek.py similarity index 100% rename from samples/spacegeek/spacegeek.py rename to f-ask/samples/spacegeek/spacegeek.py diff --git a/samples/spacegeek/speech_assets/IntentSchema.json b/f-ask/samples/spacegeek/speech_assets/IntentSchema.json similarity index 100% rename from samples/spacegeek/speech_assets/IntentSchema.json rename to f-ask/samples/spacegeek/speech_assets/IntentSchema.json diff --git a/samples/spacegeek/speech_assets/SampleUtterances.txt b/f-ask/samples/spacegeek/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/spacegeek/speech_assets/SampleUtterances.txt rename to f-ask/samples/spacegeek/speech_assets/SampleUtterances.txt diff --git a/samples/spacegeek/templates.yaml b/f-ask/samples/spacegeek/templates.yaml similarity index 100% rename from samples/spacegeek/templates.yaml rename to f-ask/samples/spacegeek/templates.yaml diff --git a/samples/tidepooler/speech_assets/IntentSchema.json b/f-ask/samples/tidepooler/speech_assets/IntentSchema.json similarity index 100% rename from samples/tidepooler/speech_assets/IntentSchema.json rename to f-ask/samples/tidepooler/speech_assets/IntentSchema.json diff --git a/samples/tidepooler/speech_assets/SampleUtterances.txt b/f-ask/samples/tidepooler/speech_assets/SampleUtterances.txt similarity index 100% rename from samples/tidepooler/speech_assets/SampleUtterances.txt rename to f-ask/samples/tidepooler/speech_assets/SampleUtterances.txt diff --git a/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_CITIES b/f-ask/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_CITIES similarity index 100% rename from samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_CITIES rename to f-ask/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_CITIES diff --git a/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_STATES b/f-ask/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_STATES similarity index 100% rename from samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_STATES rename to f-ask/samples/tidepooler/speech_assets/customSlotTypes/LIST_OF_STATES diff --git a/samples/tidepooler/templates.yaml b/f-ask/samples/tidepooler/templates.yaml similarity index 100% rename from samples/tidepooler/templates.yaml rename to f-ask/samples/tidepooler/templates.yaml diff --git a/samples/tidepooler/tidepooler.py b/f-ask/samples/tidepooler/tidepooler.py similarity index 100% rename from samples/tidepooler/tidepooler.py rename to f-ask/samples/tidepooler/tidepooler.py diff --git a/setup.cfg b/f-ask/setup.cfg similarity index 100% rename from setup.cfg rename to f-ask/setup.cfg diff --git a/setup.py b/f-ask/setup.py similarity index 100% rename from setup.py rename to f-ask/setup.py diff --git a/tests/__init__.py b/f-ask/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to f-ask/tests/__init__.py diff --git a/tests/test_audio.py b/f-ask/tests/test_audio.py similarity index 100% rename from tests/test_audio.py rename to f-ask/tests/test_audio.py diff --git a/tests/test_cache.py b/f-ask/tests/test_cache.py similarity index 100% rename from tests/test_cache.py rename to f-ask/tests/test_cache.py diff --git a/tests/test_core.py b/f-ask/tests/test_core.py similarity index 100% rename from tests/test_core.py rename to f-ask/tests/test_core.py diff --git a/tests/test_integration.py b/f-ask/tests/test_integration.py similarity index 100% rename from tests/test_integration.py rename to f-ask/tests/test_integration.py diff --git a/tests/test_integration_support_entity_resolution.py b/f-ask/tests/test_integration_support_entity_resolution.py similarity index 100% rename from tests/test_integration_support_entity_resolution.py rename to f-ask/tests/test_integration_support_entity_resolution.py diff --git a/tests/test_samples.py b/f-ask/tests/test_samples.py similarity index 100% rename from tests/test_samples.py rename to f-ask/tests/test_samples.py diff --git a/tests/test_unicode.py b/f-ask/tests/test_unicode.py similarity index 100% rename from tests/test_unicode.py rename to f-ask/tests/test_unicode.py diff --git a/tox.ini b/f-ask/tox.ini similarity index 100% rename from tox.ini rename to f-ask/tox.ini diff --git a/reader/main.py b/reader/main.py new file mode 100644 index 0000000..e69de29 diff --git a/reader/site.py b/reader/site.py new file mode 100644 index 0000000..4de01d2 --- /dev/null +++ b/reader/site.py @@ -0,0 +1,61 @@ +import urllib.request,urllib.parse,urllib.error +from lxml import html +import requests +import re + + +class site: + url = "" + header_values = { + 'Connection:' : 'Keep-alive', + 'name' : 'Michael Foord', + 'location' : 'Northampton', + 'language' : 'German', + 'User-Agent': 'Mozilla 4/0'} + + + def __init__(self, url): + self.url = urllib.request.urlopen(url).geturl() + + def search_article(self, topic): + return False + def get_news(self): + return False + def read_article(self, url): + return False + def read_headlines(self, url): + return False + + +class Golem(site): + def search_article(self, topic): + searchURL = "https://suche.golem.de/search.php?l=10&q=" + topic.replace(" ", "+") + site = requests.get(searchURL, headers=self.header_values) + tree = html.fromstring(site.content) + + articles = tree.xpath('//span[@class="dh2 head2"]/text()') + return articles + + def get_news(self): + searchURL = "https://www.golem.de/" + site = requests.get(searchURL, headers=self.header_values) + tree = html.fromstring(site.content) + + articles = tree.xpath('//h2[@class="head2"]/text()') + return articles + + def read_headlines(self, url): + site = requests.get(url, headers=self.header_values) + tree = html.fromstring(site.content) + + title = tree.xpath('//header/h1/span[@class="dh1 head5"]/text()') + title += tree.xpath('//header/p/text()') + return title + + def read_article(self, url): + site = requests.get(url, headers=self.header_values) + tree = html.fromstring(site.content) + + title = self.read_headlines(url) + title += tree.xpath('//div[@class="formatted"]/p/text()') + return title \ No newline at end of file diff --git a/reader/tests.py b/reader/tests.py new file mode 100644 index 0000000..da7566b --- /dev/null +++ b/reader/tests.py @@ -0,0 +1,10 @@ +import urllib.request,urllib.parse,urllib.error +from lxml import html +import requests +import re + +url="https://www.golem.de/news/tchap-forscher-gelingt-anmeldung-im-regierungschat-frankreichs-1904-140799.html" +site = requests.get(url) +tree = html.fromstring(site.content) +title = tree.xpath('//div[@class="formatted"]/p/text()') +print(title) \ No newline at end of file diff --git a/reader/urlchecker.py b/reader/urlchecker.py new file mode 100644 index 0000000..28311e9 --- /dev/null +++ b/reader/urlchecker.py @@ -0,0 +1,135 @@ +import urllib.request,urllib.parse,urllib.error +from urllib.error import HTTPError +from urllib.error import URLError +from urllib.parse import urljoin +import requests +import re + +import networkx as nx +import matplotlib.pyplot as plt + +class url: + + url = "" # the url of the website to be checked + sites = dict() # dic. with all sites and urls on those sites + does_work = [] # array with all prev. positiv tested urls + does_not_work = dict() # dic. with all not working urls and the site that linked there + header_values = { + 'Connection:' : 'Keep-alive', + 'name' : 'Michael Foord', + 'location' : 'Northampton', + 'language' : 'English', + 'User-Agent': 'Mozilla 4/0'} + + def __init__(self, url): + self.url = urllib.request.urlopen(url).geturl() + + + def make_url(self, link, start): + ret_link = urljoin(start, link) + + return ret_link + + def test_url(self, link, root): + + if link in self.sites or link in self.does_work: + return True + elif link in self.does_not_work: + return False + else: + try: + + header = urllib.parse.urlencode(self.header_values) + header=header.encode('ascii') + request = urllib.request.Request(link, header) + response = urllib.request.urlopen(request) + self.does_work.append(link) + print(" works " + link) + return True + + except (urllib.error.HTTPError, urllib.error.URLError, ValueError): + self.does_not_work[link]=root + print(" doesn't work " + link) + return False + + def get_actual_urls(self, links, root): + temp_links = [] + for each_link in links: + + if each_link.startswith("http") | each_link.startswith("//"): + temp_links.append(each_link) + else: + temp_links.append(urljoin(root, each_link)) + + for each_temp_link in temp_links: + self.test_url(each_temp_link, root) + + return temp_links + + def run_check(self, root=None): # root is the url of the current Site + + if root == None: + root = self.url + else: + pass + + if root in self.sites or self.url.rsplit('/', 1)[0] not in root or not self.test_url(root, root): + return + + header = urllib.parse.urlencode(self.header_values) + header=header.encode('ascii') + + request = urllib.request.Request(root, header) + http_response = urllib.request.urlopen(request) + root = http_response.geturl() + response_data= http_response.read() + + + links = re.findall(r'href="(.*?)"' , str(response_data)) + + links = self.get_actual_urls(links, root) + + + self.sites[root]=links + for each_link in links: + self.run_check(each_link) + + + def graph(self): + + G = nx.Graph(self.sites) + + label_dict = {} + for key, value in self.sites.items(): #that's not how it works... todo: later + label_dict[key]=self.remove_root(value) + + nx.draw(G, with_labels=True, font_size=8 , node_size=1000, node_color="skyblue", edge_color='#A0FFA2', pos=nx.spring_layout(G)) + + plt.show() + + def remove_root(self, links): + ret_links = [] + for link in links: + ret_links.append(link.rsplit('.', 1)[0]) + + return ret_links + + def clean(self): + self.sites.clear() + self.does_not_work.clear() + self.does_work.clear() + + + + + + + + + + + + + + + \ No newline at end of file