Assignment 1

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.pythonPath": "C:\\Users\\brhac\\.virtualenvs\\DS-Unit-3-Sprint-3-Productization-and-Clou-lU37dm1B\\Scripts\\python.exe",
+    "python.linting.pylintEnabled": false,
+    "python.linting.flake8Enabled": true,
+    "python.linting.enabled": true
+}

Pipfile

@@ -0,0 +1,20 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+flake8 = "*"
+
+[packages]
+flask = "*"
+flask-sqlalchemy = "*"
+flask-migrate = "*"
+python-dotenv = "*"
+requests = "*"
+basilica = "*"
+tweepy = "*"
+scikit-learn = "*"
+
+[requires]
+python_version = "3.7"

module1-web-application-development-with-flask/.gitignore

@@ -0,0 +1,7 @@
+#ignore local database stuff:
+
+migrations/
+
+*.db
+
+.env

module1-web-application-development-with-flask/instructions.md

@@ -0,0 +1,22 @@
+##Setup
+
+
+Setup and activate a virtual environment:
+
+
+Set up database 
+
+# Windows users can omit the "FLASK_APP=web_app" part...
+
+FLASK_APP=web_app flask db init #> generates app/migrations dir
+
+# run both when changing the schema:
+FLASK_APP=web_app flask db migrate #> creates the db (with "alembic_version" table)
+FLASK_APP=web_app flask db upgrade #> creates the specified tables
+
+# Usage
+
+run the flask 
+'''py
+flask run
+'''

module1-web-application-development-with-flask/web_app/__init__.py

@@ -0,0 +1,25 @@
+# web_app/__init__.py
+
+from flask import Flask
+
+from web_app.models import db, migrate
+from web_app.routes.home_routes import home_routes
+from web_app.routes.book_routes import book_routes
+
+DATABASE_URI = "sqlite:///web_app_brit.db"
+
+def create_app():
+    app = Flask(__name__)
+
+    app.config["SQLALCHEMY_DATABASE_URI"] = DATABASE_URI
+    app.config["SQLALCHEMY_TRACK_MODIFICATION"] =False
+    db.init_app(app)
+    migrate.init_app(app, db)
+
+    app.register_blueprint(home_routes)
+    app.register_blueprint(book_routes)
+    return app
+
+if __name__ == "__main__":
+    my_app = create_app()
+    my_app.run(debug=True)

module1-web-application-development-with-flask/web_app/models.py

@@ -0,0 +1,61 @@
+from flask_sqlalchemy import SQLAlchemy
+from flask_migrate import Migrate
+
+db = SQLAlchemy()
+
+migrate = Migrate()
+
+
+class Book(db.Model):
+    id = db.Column(db.Integer, primary_key=True)
+    title = db.Column(db.String(128))
+    author_id = db.Column(db.String(128))
+
+    def __repr__(self):
+        return f"<Book {self.id} {self.title}>"
+
+
+# def parse_records(database_records):
+#     """
+#     A helper method for converting a list of database record objects into a list of dictionaries, so they can be returned as JSON
+
+#     Param: database_records (a list of db.Model instances)
+
+#     Example: parse_records(User.query.all())
+
+#     Returns: a list of dictionaries, each corresponding to a record, like...
+#         [
+#             {"id": 1, "title": "Book 1"},
+#             {"id": 2, "title": "Book 2"},
+#             {"id": 3, "title": "Book 3"},
+#         ]
+#     """
+#     parsed_records = []
+#     for record in database_records:
+#         print(record)
+#         parsed_record = record.__dict__
+#         del parsed_record["_sa_instance_state"]
+#         parsed_records.append(parsed_record)
+#     return parsed_records
+
+class User(db.Model):
+    """Twitter users corresponding to Tweets in the Tweet table."""
+    id = db.Column(db.BigInteger, primary_key=True)
+    name = db.Column(db.String(15), nullable=False)
+    # Tweet IDs are ordinal ints, so can be used to fetch only more recent
+    newest_tweet_id = db.Column(db.BigInteger)
+
+    def __repr__(self):
+        return '<User {}>'.format(self.name)
+
+
+class Tweet(db.Model):
+    """Tweets and their embeddings from Basilica."""
+    id = db.Column(db.BigInteger, primary_key=True)
+    text = db.Column(db.Unicode(300))  # Allowing for full + links
+    embedding = db.Column(db.PickleType, nullable=False)
+    user_id = db.Column(db.BigInteger, db.ForeignKey('user.id'), nullable=False)
+    user = db.relationship('User', backref=db.backref('tweets', lazy=True))
+
+    def __repr__(self):
+        return '<Tweet {}>'.format(self.text)

module1-web-application-development-with-flask/web_app/routes/book_routes.py

@@ -0,0 +1,39 @@
+# web_app/routes/book_routes.py
+
+from flask import Blueprint, jsonify, request, render_template #, flash, redirect
+from web_app.models import Book, db
+book_routes = Blueprint("book_routes", __name__)
+
+@book_routes.route("/books.json")
+def list_books():
+    book_records = Book.query.all()
+    print(book_records)
+    books = parse_records(book_records) 
+
+    return jsonify(books)
+
+@book_routes.route("/books")
+def list_books_for_humans():
+    book_records = Book.query.all()
+    print(book_records)
+    books = parse_records(book_records) 
+    return render_template("books.html", message="Find books below", books=books)
+
+@book_routes.route("/books/new")
+def new_book():
+    return render_template("new_book.html")
+
+@book_routes.route("/books/create", methods=["POST"])
+def create_book():
+    print("FORM DATA:", dict(request.form))
+    new_book = Book(title=request.form["title"], author_id=request.form["author_name"])
+    db.session.add(new_book)
+    db.session.commit()
+
+
+    return jsonify({
+        "message": "BOOK CREATED OK (TODO)",
+        "book": dict(request.form)
+    })
+    #flash(f"Book '{new_book.title}' created successfully!", "success")
+    #return redirect(f"/books")

module1-web-application-development-with-flask/web_app/routes/home_routes.py

@@ -0,0 +1,18 @@
+# web_app/routes/home_routes.py
+
+from flask import Blueprint, render_template
+
+home_routes = Blueprint("home_routes", __name__)
+
+@home_routes.route("/")
+def index():
+    return render_template("prediction_form.html")
+
+@home_routes.route("/")
+def index():
+    x = 2 + 2
+    return f"Hello World! {x}"
+
+@home_routes.route("/about")
+def about():
+    return "About me"

module1-web-application-development-with-flask/web_app/routes/twitter_routes.py

@@ -0,0 +1,53 @@
+pifrom flask import Blueprint, render_template, jsonify
+from web_app.services.twitter_service import api as twitter_api_client
+from web_app.models import db, User, Tweet, parse_records
+
+twitter_routes = Blueprint("twitter_routes", __name__)
+
+@twitter_routes.route("/users/<screen_name>/fetch")
+def fetch_user(screen_name=None):
+    print(screen_name)
+
+    twitter_user = twitter_api_client.get_user(screen_name)
+    tweets = api.user_timeline(screen_name, tweet_mode="extended", count=150, exclude_replies=True, include_rts=False)
+    print("TWEETS COUNT:", len(tweets))
+    #return jsonify({"user": user._json, "tweets": [s._json for s in statuses]})
+
+
+    # get existing user from the db or initialize a new one:
+    db_user = User.query.get(twitter_user.id) or User(id=twitter_user.id)
+    db_user.screen_name = twitter_user.screen_name
+    db_user.name = twitter_user.name
+    db_user.location = twitter_user.location
+    db_user.followers_count = twitter_user.followers_count
+    db.session.add(db_user)
+    db.session.commit()
+    #return "OK"
+    breakpoint()
+
+    basilica_api = basilica_api_client()
+
+    #all_tweet_texts = [status.full_text for status in statuses]
+    embeddings = list(basilica_api_client.embed_sentences(all_tweet_texts, model="twitter"))
+    print("NUMBER OF EMBEDDINGS", len(embeddings))
+
+    # TODO: explore using the zip() function maybe...
+    #counter = 0
+    for index, status in enumerate(tweets):
+        print(index)
+        print(status.full_text)
+        print("----")
+
+        embedding = embeddings[index]
+        #print(dir(status))
+        # get existing tweet from the db or initialize a new one:
+        db_tweet = Tweet.query.get(status.id) or Tweet(id=status.id)
+        db_tweet.user_id = status.author.id # or db_user.id
+        db_tweet.full_text = status.full_text
+        db_tweet.embedding = embedding
+        db.session.add(db_tweet)
+
+    db.session.commit()
+    #breakpoint()
+    return "OK"
+    #return render_template("user.html", user=db_user, tweets=statuses) # tweets=db_tweets

module1-web-application-development-with-flask/web_app/services/basilica_service.py

@@ -0,0 +1,27 @@
+from basilica import Connection
+import os
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+API_KEY = os.getenv("BASILICA_API_KEY")
+
+connection = Connection(API_KEY)
+print("CONNECTION", type(connection))
+
+if __name__ == "__main__":
+    sentences = [
+        "This is a sentence!",
+        "This is a similar sentence!",
+        "I don't think this sentence is very similar at all...",
+    ]
+    embeddings = list(connection.embed_sentences(sentences))
+    print(embeddings)
+
+    embedding = list(connection.embed_sentence("Hello world!", model="twitter"))
+    print(type(embedding))
+    print(type(embedding[0]))
+    print(len(embedding))
+
+    breakpoint()

module1-web-application-development-with-flask/web_app/services/twitter_service.py

@@ -0,0 +1,31 @@
+import os
+from dotenv import load_dotenv
+from tweepy import OAuthHandler, API
+
+load_dotenv()
+
+TWITTER_API_KEY = os.getenv("TWITTER_API_KEY")
+TWITTER_API_SECRET = os.getenv("TWITTER_API_SECRET")
+TWITTER_ACCESS_TOKEN = os.getenv("TWITTER_ACCESS_TOKEN")
+TWITTER_ACCESS_TOKEN_SECRET = os.getenv("TWITTER_ACCESS_TOKEN_SECRET")
+
+auth = OAuthHandler(TWITTER_API_KEY, TWITTER_API_SECRET)
+auth.set_acess_token(TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET)
+
+api = API(auth)
+print("API CLIENT:", api)
+
+user = api.get_user("britnelikecafe")
+print("TWITTER USER:", type(user))
+print(user.id)
+print(user.screen_name)
+print(user.name)
+
+tweets = api.user_timeline("britnelikecafe")
+print("TWEETS", type(tweets))
+print(type(tweets[0])) 
+
+breakpoint()
+tweet = tweets[0]
+print(tweet.id)
+print(tweet.text)

module1-web-application-development-with-flask/web_app/templates/books.html

@@ -0,0 +1,28 @@
+<!-- web_app/templates/books.html -->
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Ny webapp</title>
+</head>
+<body>
+    <h2>Welcome to Bookdom!</h2>
+
+    <p>{{ message }}</p>
+
+    {% if books %}
+        <ul>
+        {% for book in books %}
+            <li>{{ book["title"] }}</li>
+        {% endfor %}
+        </ul>
+
+    {% else %}
+        <p>Books not found.</p>
+    {% endif %}   
+</body>
+</html>
+
+

module1-web-application-development-with-flask/web_app/templates/new_book.html

@@ -0,0 +1,31 @@
+<!-- web_app/templates/new_book.html -->
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>My webapp</title>
+</head>
+<body>
+    <h1>New Book Page</h1>
+
+    <p>Please fill out the form and submit to create a new book!</p>
+
+    <form action="/books/create" method="POST">
+
+        <label>Title:</label>
+        <input type="text" name="title" placeholder="Book XYZ" value="Book XYZ">
+
+        <label>Author:</label>
+        <select name="author_name">
+          <option value="A1">Author 1</option>
+          <option value="A2">Author 2</option>
+          <option value="A3">Author 3</option>
+        </select> 
+
+        <button>Submit</button>
+    </form>
+</body>
+</html>
+