Merge pull request #11 from uclatommy/release-0.2.0

Release 0.2.0, ship it!

.travis.yml

@@ -15,6 +15,7 @@ before_install:
 
 install:
   - pip3 install -r requirements.txt
+  - python3 -m nltk.downloader vader_lexicon
   - python3 setup.py install
   - pip3 install nose
 

requirements.txt

@@ -10,4 +10,4 @@ requests-oauthlib>=0.8.0
 six>=1.10.0
 tweepy>=3.5.0
 twython>=3.4.0
-vaderSentiment>=2.5
+nose

setup.py

@@ -1,12 +1,31 @@
 #from distutils.core import setup
 from setuptools import setup
 
+filename = 'tweetfeels/version.py'
+exec(compile(open(filename, "rb").read(), filename, 'exec'))
+
 setup(name='tweetfeels',
-      version='0.1.3',
+      version=__version__,
       description='Real-time sentiment analysis for twitter.',
       author='Thomas Chen',
       author_email='tkchen@gmail.com',
       url='https://github.com/uclatommy/tweetfeels',
-      download_url='https://github.com/uclatommy/tweetfeels/tarball/0.1.3',
-      packages=['tweetfeels']
+      download_url='https://github.com/uclatommy/tweetfeels/tarball/{}'.format(
+          __version__
+          ),
+      packages=['tweetfeels'],
+      classifiers=[
+          'Development Status :: 4 - Beta',
+          'License :: OSI Approved :: BSD License',
+          'Natural Language :: English',
+          'Programming Language :: Python :: 3.6',
+          'Topic :: Scientific/Engineering :: Artificial Intelligence'
+          ],
+      install_requires=[
+          'tweepy', 'h5py', 'nltk', 'numpy', 'oauthlib', 'pandas',
+          'python-dateutil', 'pytz', 'requests', 'requests-oauthlib',
+          'six', 'twython'
+          ],
+      test_suite='nose.collector',
+      tests_require=['nose']
      )

test/test_data.py

@@ -1,6 +1,7 @@
 import unittest
 from tweetfeels import TweetData
 from tweetfeels import Tweet
+from datetime import datetime
 import json
 import os
 
@@ -32,14 +33,16 @@ def test_data_operation(self):
                'id_str': '833394296418082817',
                'text': 'All the feels!'}
         t = Tweet(twt)
-        self.assertEqual(len(t.keys()), 3)
+        self.assertEqual(len(t.keys()), 7)
         self.feels_db.insert_tweet(t)
-        df = self.feels_db.queue
-        self.assertEqual(len(df), 1)
-        df.sentiment = 0.9
-        for row in df.itertuples():
-            self.feels_db.update_tweet(
-                {'id_str': row.id_str, 'sentiment': row.sentiment}
-                )
-        self.assertEqual(len(self.feels_db.queue), 0)
-        self.assertEqual(len(self.feels_db.all), 1)
+        dfs = self.feels_db.tweets_since(datetime.now())
+        for df in dfs:
+            self.assertEqual(len(df), 0)
+        dfs = self.feels_db.tweets_since(0)
+        for df in dfs:
+            self.assertEqual(len(df), 1)
+            df.sentiment = 0.9
+            for row in df.itertuples():
+                self.feels_db.update_tweet(
+                    {'id_str': row.id_str, 'sentiment': row.sentiment}
+                    )

test/test_feels.py

@@ -0,0 +1,78 @@
+import unittest
+from unittest.mock import patch, MagicMock
+import json
+import os
+import time
+
+from tweetfeels import (TweetFeels, Tweet, TweetData)
+
+
+class Test_Feels(unittest.TestCase):
+    def setUp(self):
+        TweetFeels._db_factory = (lambda db: MagicMock())
+        TweetFeels._auth_factory = (lambda cred: MagicMock())
+        TweetFeels._listener_factory = (lambda ctrl: MagicMock())
+        TweetFeels._stream_factory = (lambda auth, listener: MagicMock())
+        self.tweets_data_path = 'test/sample.json'
+
+    def test_start(self):
+        mock_feels = TweetFeels("abcd")
+        mock_feels.tracking = []
+        mock_feels.start()
+        mock_feels._stream.filter.assert_not_called()
+        mock_feels.tracking = ['tsla']
+        mock_feels.start()
+        mock_feels._stream.filter.assert_called_once()
+
+    def test_stop(self):
+        mock_feels = TweetFeels("abcd")
+        mock_feels.stop()
+        mock_feels._stream.disconnect.assert_called_once()
+
+    def test_on_data(self):
+        mock_feels = TweetFeels("abcd")
+        mock_feels.buffer_limit = 0
+        data = {'filter_level': 'low', 'text': 'test data'}
+        mock_feels.on_data(data)
+        mock_feels._feels.insert_tweet.assert_called_once()
+
+        # test filtering levels
+        mock_feels2 = TweetFeels("abcd")
+        mock_feels2._filter_level = 'medium'
+        mock_feels2.on_data(data)
+        mock_feels2._feels.insert_tweet.assert_not_called()
+
+        # test buffer limit. no inserts until we are over limit
+        mock_feels2.buffer_limit = 2
+        mock_feels2.filter_level = 'low'
+        mock_feels2.on_data(data)
+        mock_feels2._feels.insert_tweet.assert_not_called()
+        mock_feels2.on_data(data)
+        mock_feels2.on_data(data)
+        mock_feels._feels.insert_tweet.assert_called_once()
+
+    def test_sentiment(self):
+        mock_feels = TweetFeels("abcd")
+        mock_feels._feels.tweets_since = MagicMock(return_value=[])
+        mock_feels._sentiment = 0.5
+        self.assertEqual(mock_feels.sentiment, 0.5)
+
+    def test_buffer(self):
+        mock_feels = TweetFeels('abcd')
+        mock_feels.buffer_limit = 5
+        feels_db = TweetData(file='sample.sqlite')
+        mock_feels._feels = feels_db
+        with open(self.tweets_data_path) as tweets_file:
+            lines = list(filter(None, (line.rstrip() for line in tweets_file)))
+            for line in lines[0:3]:
+                t = Tweet(json.loads(line))
+                mock_feels.on_data(t)
+            self.assertEqual(len(mock_feels._tweet_buffer), 3)
+            for line in lines[3:6]:
+                t = Tweet(json.loads(line))
+                mock_feels.on_data(t)
+            time.sleep(1) #this waits for items to finish popping off the buffer
+            self.assertEqual(len(mock_feels._tweet_buffer), 0)
+            dfs = [df for df in mock_feels._feels.all]
+            self.assertEqual(len(dfs[0]), 6)
+        os.remove('sample.sqlite')

test/test_listener.py

@@ -1,19 +1,70 @@
 import unittest
+from unittest.mock import patch, MagicMock
+
 from tweetfeels import TweetListener
 from tweetfeels import Tweet
+
 import json
+from datetime import datetime
 
 
 class Test_Listener(unittest.TestCase):
     def setUp(self):
         self.tweets_data_path = 'test/sample.json'
+        self.disconnect_msg = """
+            {
+              "disconnect":{
+                "code": 4,
+                "stream_name":"",
+                "reason":""
+              }
+            }
+            """
 
-    def test_listener(self):
-        tl = TweetListener(None, None)
+    @patch('tweetfeels.TweetFeels')
+    def test_listener(self, mock_feels):
+        tl = TweetListener(mock_feels)
         with open(self.tweets_data_path) as tweets_file:
             lines = filter(None, (line.rstrip() for line in tweets_file))
             for line in lines:
-                self.assertTrue(tl.on_data(line))
+                tl.on_data(line)
+                mock_feels.on_data.assert_called()
+
+    @patch('tweetfeels.TweetFeels')
+    def test_on_disconnect(self, mock_feels):
+        tl = TweetListener(mock_feels)
+        tl.reconnect_wait = MagicMock()
+        tl.on_disconnect(self.disconnect_msg)
+        tl.reconnect_wait.assert_called_with('linear')
+        tl._controller.start.assert_called_once()
+
+    @patch('tweetfeels.TweetFeels')
+    def test_on_connect(self, mock_feels):
+        tl = TweetListener(mock_feels)
+        tl.waited = 60
+        tl.on_connect()
+        self.assertEqual(tl.waited, 0)
+
+    @patch('tweetfeels.TweetFeels')
+    def test_on_error(self, mock_feels):
+        tl = TweetListener(mock_feels)
+        tl.reconnect_wait = MagicMock()
+        tl.on_error(420)
+        tl.reconnect_wait.assert_called_with('exponential')
+        self.assertEqual(tl.waited, 60)
+        mock_feels.on_error.assert_called_with(420)
+
+    @patch('tweetfeels.TweetFeels')
+    def test_reconnect_wait(self, mock_feels):
+        tl = TweetListener(mock_feels)
+        tl.waited = 0.1
+        tl.reconnect_wait('linear')
+        self.assertEqual(tl.waited, 1.1)
+        tl.waited = 0.1
+        tl.reconnect_wait('exponential')
+        tl.reconnect_wait('exponential')
+        self.assertEqual(tl.waited, 0.4)
+
 
 class Test_Tweet(unittest.TestCase):
     def setUp(self):
@@ -27,8 +78,16 @@ def test_tweet_keys(self):
         self.assertEqual(self.tweet['followers_count'], 83)
         self.assertEqual(self.tweet['friends_count'], 303)
         self.assertTrue(len(self.tweet)>0)
-        self.assertEqual(self.tweet['created_at'], '2017-02-19 19:14:18')
+        dt = datetime(2017, 2, 19, 19, 14, 18)
+        self.assertEqual(self.tweet['created_at'], dt)
 
     def test_attributes(self):
-        self.assertEqual(len(self.tweet), 29)
+        self.assertEqual(len(self.tweet), 33)
         self.assertTrue('followers_count' in self.tweet)
+        self.assertTrue(isinstance(self.tweet['created_at'], datetime))
+
+    def test_sentiment(self):
+        self.assertEqual(self.tweet.sentiment['compound'], -0.2472)
+        self.assertEqual(self.tweet.sentiment['pos'], 0.087)
+        self.assertEqual(self.tweet.sentiment['neu'], 0.752)
+        self.assertEqual(self.tweet.sentiment['neg'], 0.161)

tweetfeels/tweetdata.py

@@ -12,6 +12,7 @@ def __init__(self, file='feels.sqlite'):
         if not os.path.isfile(self._db):
             self.make_feels_db(self._db)
         self._debug = False
+        self.chunksize=1000
 
     @property
     def fields(self):
@@ -22,18 +23,21 @@ def fields(self):
         c.close()
         return fields
 
-    @property
-    def queue(self):
-        conn = sqlite3.connect(self._db)
+    def tweets_since(self, dt):
+        conn = sqlite3.connect(self._db, detect_types=sqlite3.PARSE_DECLTYPES)
         df = pd.read_sql_query(
-            'SELECT * FROM tweets WHERE sentiment is NULL', conn
+            'SELECT * FROM tweets WHERE created_at > ?', conn, params=(dt,),
+            parse_dates=['created_at'], chunksize=self.chunksize
             )
         return df
 
     @property
     def all(self):
-        conn = sqlite3.connect(self._db)
-        df = pd.read_sql_query('SELECT * FROM tweets', conn)
+        conn = sqlite3.connect(self._db, detect_types=sqlite3.PARSE_DECLTYPES)
+        df = pd.read_sql_query(
+            'SELECT * FROM tweets', conn, parse_dates=['created_at'],
+            chunksize=self.chunksize
+            )
         return df
 
     def make_feels_db(self, filename='feels.sqlite'):
@@ -42,7 +46,7 @@ def make_feels_db(self, filename='feels.sqlite'):
         tbl_def = 'CREATE TABLE tweets(\
         id_str          CHARACTER(20)  PRIMARY KEY NOT NULL,\
         text            CHARACTER(140)             NOT NULL,\
-        created_at      TEXT                       NOT NULL,\
+        created_at      timestamp                  NOT NULL,\
         coordinates     VARCHAR(20),\
         favorite_count  INTEGER,\
         favorited       VARCHAR(5),\
@@ -53,7 +57,10 @@ def make_feels_db(self, filename='feels.sqlite'):
         friends_count   INTEGER,\
         followers_count INTEGER,\
         location        TEXT,\
-        sentiment       DOUBLE\
+        sentiment       DOUBLE                     NOT NULL,\
+        pos             DOUBLE                     NOT NULL,\
+        neu             DOUBLE                     NOT NULL,\
+        neg             DOUBLE                     NOT NULL\
         )'
         c.execute(tbl_def)
         c.close()
@@ -72,7 +79,9 @@ def insert_tweet(self, tweet):
         ins = ins + '?)'
         qry = f'INSERT OR IGNORE INTO tweets {keys} VALUES {ins}'
         try:
-            conn = sqlite3.connect(self._db)
+            conn = sqlite3.connect(
+                self._db, detect_types=sqlite3.PARSE_DECLTYPES
+                )
             c = conn.cursor()
             c.execute(qry, vals)
             c.close()
@@ -95,7 +104,9 @@ def update_tweet(self, tweet):
 
         qry = f'UPDATE tweets SET {updt} WHERE id_str=?'
         try:
-            conn = sqlite3.connect(self._db)
+            conn = sqlite3.connect(
+                self._db, detect_types=sqlite3.PARSE_DECLTYPES
+                )
             c = conn.cursor()
             c.execute(qry, vals+(id_str,))
             c.close()