Surf Data
# Howard Wang 06/10/19
# CSE 145: Embedded systems and design
# Main Database: https://surf.smartfin.org/
# Analyzing data from real world categorized surf session data,
# Data is labeled into PUSHING BOARD INTO WATER',
#'SYNC (FLIP BOARD UPSIDE DOWN TO SYNC DATA/FOOTAGE)',
#'FLIP BOARD RIGHT SIDE UP',
#'WALKING IN WATER',
#'PUSH-OFF',
#'PADDLING INTO WAVES',
#'SIT-UP',
#'FLOATING',
#"TURNING TO SURFER'S LEFT",
#'LAY-DOWN',
#'PADDLING FOR A WAVE',
#'POP-UP',
#'SURFING',
#'STEP-OFF',
#"TURNING TO SURFER'S RIGHT",
#'SIT-BACK',
#'OFF-BOARD',
#'PADDLING',
#'WIPE-OUT',
#'PULL-BACK LEASH',
#'PADDLING FOR POSITION',
#'NEW',
#'DONE, OUT OF WATER',
#'WALKING OUT OF WATER'
# MATPLOTLIB
import matplotlib
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.animation as animation
#from mpl_toolkits.basemap import Basemap
# DATAFRAMES
import pandas as pd
import numpy as np
# SCIPY
from scipy import stats
from scipy import constants
from scipy import signal #added
from scipy.interpolate import CubicSpline
from scipy.interpolate import interp1d
from scipy.integrate import simps
from scipy.integrate import cumtrapz
import pylab as pylab
# SYSTEM and CONVERSION TOOLS
import math
import abc
import sys
import csv
import io
import os
import datetime
import pytz
import re
# MODELING AND GRAPHS
import peakutils
import statsmodels.api as sm
# URL REQUESTS
import requests
# VECTORS AND GRAPHICS
import mpld3
import folium
# import cmocean
import skinematics as skin
from skinematics import quat, vector, misc, rotmat, imus, view
import pygame
# PLOTTING TOOLS
from plotly import tools
import plotly.offline
import plotly.graph_objs as go
plt.rc("font", size=14)
#Scikit Learn
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
import seaborn as sns
sns.set(style="white")
sns.set(style="whitegrid", color_codes=True)
%matplotlib notebook
%matplotlib inline
print("Done!")Done!
# 14743 - Motion Control July 10th
# 14750 - Magnetometer Control July 11th
# 14814 - Pool Displacement Control July 17th
# 14815 - Compass Orientation (Lying on Charger Side) July 19th
# 14816 - Orientation w Higher Sampling (Lying on Charger Side) July 20th
# 14827 - Pool Displacement Control w Higher Sampling (Jul 23)
# 14888 - First Buoy Calibration Experiment (July 30)
# 15218 - Jasmine's Second Ride Sesh filmed with GoPro (Aug 29) //no footage
# 15629 - Jasmine's First Ride Sesh filmed with VIRB (Oct. 24) //first labelled footage!
# 15669 - Jasmine's Second Ride Sesh filmed with VIRB (Nov. 7) //second labelled footage!
# 15692 - Jasmine's 3rd Ride Sesh filmed with VIRB (Nov. 9) //third labelled footage!
# 15686 - Jasmine's 4th Ride Sesh filmed with VIRB (Nov. 11) //fourth labelled footage!
ride_ids = ['15692']
print("Done!")Done!
#%% Fin ID scraper
# Input fin ID, get all ride IDs
# base URL to which we'll append given fin IDs
fin_url_base = 'http://surf.smartfin.org/fin/'
# Look for the following text in the HTML contents in fcn below
str_id_ride = 'rideId = \'' # backslash allows us to look for single quote
str_id_date = 'var date = \'' # backslash allows us to look for single quote
#%% Ride ID scraper
# Input ride ID, get ocean and motion CSVs
# Base URL to which we'll append given ride IDs
ride_url_base = 'https://surf.smartfin.org/ride/'
# Look for the following text in the HTML contents in fcn below
str_id_csv = 'img id="temperatureChart" class="chart" src="'
def get_csv_from_ride_id(rid):
# Build URL for each individual ride
ride_url = ride_url_base+str(rid)
print(ride_url)
# Get contents of ride_url
html_contents = requests.get(ride_url).text
# Find CSV identifier
loc_csv_id = html_contents.find(str_id_csv)
# Different based on whether user logged in with FB or Google
offset_googleOAuth = [46, 114]
offset_facebkOAuth = [46, 112]
if html_contents[loc_csv_id+59] == 'f': # Facebook login
off0 = offset_facebkOAuth[0]
off1 = offset_facebkOAuth[1]
else: # Google login
off0 = offset_googleOAuth[0]
off1 = offset_googleOAuth[1]
csv_id_longstr = html_contents[loc_csv_id+off0:loc_csv_id+off1]
# Stitch together full URL for CSV
if ("media" in csv_id_longstr) & ("Calibration" not in html_contents): # other junk URLs can exist and break everything
ocean_csv_url = 'https://surf.smartfin.org/'+csv_id_longstr+'Ocean.CSV'
motion_csv_url = 'https://surf.smartfin.org/'+csv_id_longstr+'Motion.CSV'
print(ocean_csv_url)
# Go to ocean_csv_url and grab contents (theoretically, a CSV)
oceanCSV = requests.get(ocean_csv_url).content
ocean_df_small = pd.read_csv(io.StringIO(oceanCSV.decode('utf-8')))
# Grab CSV from motion url
motionCSV = requests.get(motion_csv_url).content
motion_df_small = pd.read_csv(io.StringIO(motionCSV.decode('utf-8')))
return ocean_df_small, motion_df_small
else:
ocean_df_small_resample = pd.DataFrame() # empty DF just so something is returned
motion_df_small_resample = pd.DataFrame()
return ocean_df_small_resample, motion_df_small_resample
print("Done!")Done!
appended_ocean_list = [] # list of DataFrames from original CSVs
appended_motion_list = []
appended_multiIndex = [] # fin_id & ride_id used to identify each DataFrame
## Nested loops (for each fin ID, find all ride IDs, then build a DataFrame from all ride CSVs)
## (Here, ride IDS are either ocean or motion dataframes)
count_good_fins = 0
# Loop over ride_ids and find CSVs
for rid in ride_ids:
try:
new_ocean_df, new_motion_df = get_csv_from_ride_id(rid) # get given ride's CSV from its ride ID using function above
#print(len(new_ocean_df))
#print(len(new_motion_df))
if not new_ocean_df.empty: # Calibration rides, for example
# Append only if DF isn't empty. There may be a better way to control empty DFs which are created above
appended_multiIndex.append(str(rid)) # build list to be multiIndex of future DataFrame
appended_ocean_list.append(new_ocean_df)
appended_motion_list.append(new_motion_df)
print("Ride data has been uploaded.")
#print("Ride: ", rid, "data has been uploaded.")
count_good_fins += 1
except:
print("Ride threw an exception!")
#print("Ride ", rid, "threw an exception!")
#%% Build the "Master" DataFrame
# appended_ocean_df.summary()
df_keys = tuple(appended_multiIndex) # keys gotta be a tuple, a list which data in it cannot be changed
ocean_df = pd.concat(appended_ocean_list, keys = df_keys, names=['ride_id'])
motion_df = pd.concat(appended_motion_list, keys = df_keys, names = ['ride_id'])
print("done!")https://surf.smartfin.org/ride/15692
https://surf.smartfin.org/media/201811/google_105349665704999793400_0006667E229D_181109191556_Ocean.CSV
Ride data has been uploaded.
done!
#Drop the latitude and longitude values since most of them are Nan:
motion_df_dropped = motion_df.drop(columns=['Latitude', 'Longitude'])
#Drop the NAN values from the motion data:
motion_df_dropped = motion_df_dropped.dropna(axis=0, how='any')
print(motion_df_dropped) UTC Time IMU A1 IMU A2 \
ride_id
15692 1 2018-11-09T19:16:03.8090+00:00 1414742887 493.0 48.0
2 2018-11-09T19:16:04.0610+00:00 1414743138 513.0 89.0
3 2018-11-09T19:16:04.3120+00:00 1414743387 494.0 92.0
4 2018-11-09T19:16:04.5650+00:00 1414743639 421.0 205.0
5 2018-11-09T19:16:04.8170+00:00 1414743889 534.0 306.0
6 2018-11-09T19:16:05.0680+00:00 1414744139 455.0 149.0
7 2018-11-09T19:16:05.3210+00:00 1414744390 474.0 342.0
8 2018-11-09T19:16:05.5730+00:00 1414744641 363.0 323.0
9 2018-11-09T19:16:05.8260+00:00 1414744892 -21.0 510.0
10 2018-11-09T19:16:06.0790+00:00 1414745144 35.0 283.0
11 2018-11-09T19:16:06.3300+00:00 1414745393 130.0 323.0
12 2018-11-09T19:16:06.5820+00:00 1414745644 -55.0 504.0
13 2018-11-09T19:16:06.8350+00:00 1414745895 58.0 420.0
14 2018-11-09T19:16:07.0870+00:00 1414746146 -179.0 454.0
15 2018-11-09T19:16:07.3300+00:00 1414746387 13.0 181.0
...
[21645 rows x 11 columns]
#Create an elapsed_timedelta field:
#timedelta_values = (motion_df_dropped['Time']-motion_df_dropped['Time'][0])
#motion_df_dropped.insert(loc=1, column='TimeDelta', value=timedelta_values, drop=True)
motion_df_dropped['TimeDelta'] = (motion_df_dropped['Time']-motion_df_dropped['Time'][0])
#print(elapsed_timedelta)
#motion_df_dropped.head()
motion_df_dropped.head(10)
<style scoped>
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
</style>
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| UTC | Time | IMU A1 | IMU A2 | IMU A3 | IMU G1 | IMU G2 | IMU G3 | IMU M1 | IMU M2 | IMU M3 | TimeDelta | ||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ride_id | |||||||||||||
| 15692 | 1 | 2018-11-09T19:16:03.8090+00:00 | 1414742887 | 493.0 | 48.0 | 110.0 | 75.0 | -124.0 | -86.0 | -309.0 | 209.0 | 39.0 | 0 |
| 2 | 2018-11-09T19:16:04.0610+00:00 | 1414743138 | 513.0 | 89.0 | 62.0 | 34.0 | -36.0 | -92.0 | -320.0 | 194.0 | 38.0 | 251 | |
| 3 | 2018-11-09T19:16:04.3120+00:00 | 1414743387 | 494.0 | 92.0 | 80.0 | 69.0 | -63.0 | -42.0 | -329.0 | 189.0 | 49.0 | 500 | |
| 4 | 2018-11-09T19:16:04.5650+00:00 | 1414743639 | 421.0 | 205.0 | -104.0 | 192.0 | -92.0 | -37.0 | -330.0 | 180.0 | 64.0 | 752 | |
| 5 | 2018-11-09T19:16:04.8170+00:00 | 1414743889 | 534.0 | 306.0 | -32.0 | -421.0 | -233.0 | -229.0 | -325.0 | 161.0 | 97.0 | 1002 | |
| 6 | 2018-11-09T19:16:05.0680+00:00 | 1414744139 | 455.0 | 149.0 | -102.0 | -355.0 | -376.0 | -397.0 | -337.0 | 117.0 | 151.0 | 1252 | |
| 7 | 2018-11-09T19:16:05.3210+00:00 | 1414744390 | 474.0 | 342.0 | -219.0 | -234.0 | -527.0 | -465.0 | -311.0 | 25.0 | 217.0 | 1503 | |
| 8 | 2018-11-09T19:16:05.5730+00:00 | 1414744641 | 363.0 | 323.0 | -131.0 | 60.0 | -662.0 | -305.0 | -238.0 | -8.0 | 272.0 | 1754 | |
| 9 | 2018-11-09T19:16:05.8260+00:00 | 1414744892 | -21.0 | 510.0 | -447.0 | 78.0 | -643.0 | -153.0 | -159.0 | -21.0 | 321.0 | 2005 | |
| 10 | 2018-11-09T19:16:06.0790+00:00 | 1414745144 | 35.0 | 283.0 | -132.0 | -114.0 | -430.0 | 132.0 | -86.0 | -38.0 | 326.0 | 2257 |
#Footage sync code written by Alina: (Miulti-Column)
import time
#simple method: only walking, paddling, floating, surfing
#complex method: columns created based on footage file labels
def label_data( footage_file = '../../Footage/Footage.txt', labelling_method = 'simple', sync_threshold = 20000 ):
#First, perform sync
sync_buf = 0
with open(footage_file) as file:
for line in file:
labelled_time = line.split(None, 2)
try:
cur_time = time.strptime(labelled_time[0], '%M:%S')
except:
continue
labelled_time[1] = labelled_time[1].rstrip()
if labelled_time[1].lower() == 'sync': #Assumption that first word in sync line is "sync"
sync_time = cur_time.tm_min * 60 * 1000 + cur_time.tm_sec * 1000
index = 0
start = 0
end = 0
#Syncing occurs when IMU A2 data is negative for a longer period than the provided threshold
#Default is 20 seconds
for data in motion_df_dropped['IMU A2']:
if data < 0 and start == 0:
start = motion_df_dropped['TimeDelta'][index]
elif data > 0 and start != 0:
end = motion_df_dropped['TimeDelta'][index]
if end - start > sync_threshold:
sync_buf = start - sync_time
break
start = 0
index += 1
accepted_labels = set()
if labelling_method == 'simple':
accepted_labels = {'WALKING', 'PADDLING', 'FLOATING', 'SURFING'}
#Create new DataFrame containing label info
label_frame = pd.DataFrame(0, index = motion_df_dropped.index, columns = accepted_labels)
for label in accepted_labels:
label_frame[label] = [0] * len(motion_df_dropped['Time'])
#Convention of labelled footage text: "MINUTE:SECOND LABEL"
elapsed_time = 0
cur_label = ''
buffer = 0
with open(footage_file) as file:
for line in file:
if labelling_method == 'simple':
labelled_time = line.split(None, 2) #simple categorizes on a one-word basis
else:
labelled_time = line.split(None, 1) #complex requires the entire label
#If the first word is not a properly formatted time, the line cannot be read
try:
cur_time = time.strptime(labelled_time[0], '%M:%S')
cur_timeMS = cur_time.tm_min * 60 * 1000 + cur_time.tm_sec * 1000 + sync_buf
except:
continue
labelled_time[1] = labelled_time[1].rstrip() #Remove potential newline
#Check for end of video and modify buffer accordingly
if labelled_time[1].lower() == 'end of video': #Assumption that label end video with "end of video"
buffer += cur_timeMS
#----Complex "mode" below: --------
#Modify accepted labels list if reading a new label and in complex mode
elif labelling_method == 'complex' and (labelled_time[1].upper() not in accepted_labels):
accepted_labels.add(labelled_time[1].upper())
if not cur_label:
label_frame = pd.DataFrame(0, index = motion_df_dropped.index, columns = accepted_labels)
label_frame[labelled_time[1].upper()] = [0] * len(motion_df_dropped['Time'])
if labelled_time[1].upper() in accepted_labels:
while (elapsed_time < len(motion_df_dropped['Time']) and
(np.isnan(motion_df_dropped['TimeDelta'][elapsed_time]) or
motion_df_dropped['TimeDelta'][elapsed_time] < cur_timeMS + buffer)):
if cur_label != '':
label_frame[cur_label][elapsed_time] = 1
elapsed_time += 1
if labelled_time[1].upper() != 'end of video':
cur_label = labelled_time[1].upper()
labelled = pd.concat([motion_df_dropped, label_frame], axis = 1)
return labelled
pd.options.display.max_rows = 5000
pd.options.display.max_columns = 5000motion_df_simple = label_data('../../Footage/Footage3.txt', 'simple')
motion_df_simple.head(10)
<style scoped>
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
</style>
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| UTC | Time | IMU A1 | IMU A2 | IMU A3 | IMU G1 | IMU G2 | IMU G3 | IMU M1 | IMU M2 | IMU M3 | TimeDelta | SURFING | PADDLING | WALKING | FLOATING | ||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ride_id | |||||||||||||||||
| 15692 | 1 | 2018-11-09T19:16:03.8090+00:00 | 1414742887 | 493.0 | 48.0 | 110.0 | 75.0 | -124.0 | -86.0 | -309.0 | 209.0 | 39.0 | 0 | 0 | 0 | 0 | 0 |
| 2 | 2018-11-09T19:16:04.0610+00:00 | 1414743138 | 513.0 | 89.0 | 62.0 | 34.0 | -36.0 | -92.0 | -320.0 | 194.0 | 38.0 | 251 | 0 | 0 | 0 | 0 | |
| 3 | 2018-11-09T19:16:04.3120+00:00 | 1414743387 | 494.0 | 92.0 | 80.0 | 69.0 | -63.0 | -42.0 | -329.0 | 189.0 | 49.0 | 500 | 0 | 0 | 0 | 0 | |
| 4 | 2018-11-09T19:16:04.5650+00:00 | 1414743639 | 421.0 | 205.0 | -104.0 | 192.0 | -92.0 | -37.0 | -330.0 | 180.0 | 64.0 | 752 | 0 | 0 | 0 | 0 | |
| 5 | 2018-11-09T19:16:04.8170+00:00 | 1414743889 | 534.0 | 306.0 | -32.0 | -421.0 | -233.0 | -229.0 | -325.0 | 161.0 | 97.0 | 1002 | 0 | 0 | 0 | 0 | |
| 6 | 2018-11-09T19:16:05.0680+00:00 | 1414744139 | 455.0 | 149.0 | -102.0 | -355.0 | -376.0 | -397.0 | -337.0 | 117.0 | 151.0 | 1252 | 0 | 0 | 0 | 0 | |
| 7 | 2018-11-09T19:16:05.3210+00:00 | 1414744390 | 474.0 | 342.0 | -219.0 | -234.0 | -527.0 | -465.0 | -311.0 | 25.0 | 217.0 | 1503 | 0 | 0 | 0 | 0 | |
| 8 | 2018-11-09T19:16:05.5730+00:00 | 1414744641 | 363.0 | 323.0 | -131.0 | 60.0 | -662.0 | -305.0 | -238.0 | -8.0 | 272.0 | 1754 | 0 | 0 | 0 | 0 | |
| 9 | 2018-11-09T19:16:05.8260+00:00 | 1414744892 | -21.0 | 510.0 | -447.0 | 78.0 | -643.0 | -153.0 | -159.0 | -21.0 | 321.0 | 2005 | 0 | 0 | 0 | 0 | |
| 10 | 2018-11-09T19:16:06.0790+00:00 | 1414745144 | 35.0 | 283.0 | -132.0 | -114.0 | -430.0 | 132.0 | -86.0 | -38.0 | 326.0 | 2257 | 0 | 0 | 0 | 0 |
motion_df_complex = label_data('../../Footage/Footage3.txt', 'complex')
motion_df_complex.head(10)
<style scoped>
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
</style>
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| UTC | Time | IMU A1 | IMU A2 | IMU A3 | IMU G1 | IMU G2 | IMU G3 | IMU M1 | IMU M2 | IMU M3 | TimeDelta | SYNC | FLIP BOARD RIGHT SIDE UP | WALKING IN WATER | PUSH-OFF | PADDLING INTO WAVES | SIT-UP | FLOATING | TURNING TO SURFER'S LEFT | LAY-DOWN | PADDLING FOR A WAVE | POP-UP | SURFING | STEP-OFF | TURNING TO SURFER'S RIGHT | SIT-BACK | OFF-BOARD | PADDLING | WIPE-OUT | PULL-BACK LEASH | PADDLING FOR POSITION | NEW | DISCARD | DONE, OUT OF WATER | WALKING OUT OF WATER | ||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ride_id | |||||||||||||||||||||||||||||||||||||
| 15692 | 1 | 2018-11-09T19:16:03.8090+00:00 | 1414742887 | 493.0 | 48.0 | 110.0 | 75.0 | -124.0 | -86.0 | -309.0 | 209.0 | 39.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 2 | 2018-11-09T19:16:04.0610+00:00 | 1414743138 | 513.0 | 89.0 | 62.0 | 34.0 | -36.0 | -92.0 | -320.0 | 194.0 | 38.0 | 251 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 3 | 2018-11-09T19:16:04.3120+00:00 | 1414743387 | 494.0 | 92.0 | 80.0 | 69.0 | -63.0 | -42.0 | -329.0 | 189.0 | 49.0 | 500 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 4 | 2018-11-09T19:16:04.5650+00:00 | 1414743639 | 421.0 | 205.0 | -104.0 | 192.0 | -92.0 | -37.0 | -330.0 | 180.0 | 64.0 | 752 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 5 | 2018-11-09T19:16:04.8170+00:00 | 1414743889 | 534.0 | 306.0 | -32.0 | -421.0 | -233.0 | -229.0 | -325.0 | 161.0 | 97.0 | 1002 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 6 | 2018-11-09T19:16:05.0680+00:00 | 1414744139 | 455.0 | 149.0 | -102.0 | -355.0 | -376.0 | -397.0 | -337.0 | 117.0 | 151.0 | 1252 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 7 | 2018-11-09T19:16:05.3210+00:00 | 1414744390 | 474.0 | 342.0 | -219.0 | -234.0 | -527.0 | -465.0 | -311.0 | 25.0 | 217.0 | 1503 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 8 | 2018-11-09T19:16:05.5730+00:00 | 1414744641 | 363.0 | 323.0 | -131.0 | 60.0 | -662.0 | -305.0 | -238.0 | -8.0 | 272.0 | 1754 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 9 | 2018-11-09T19:16:05.8260+00:00 | 1414744892 | -21.0 | 510.0 | -447.0 | 78.0 | -643.0 | -153.0 | -159.0 | -21.0 | 321.0 | 2005 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 10 | 2018-11-09T19:16:06.0790+00:00 | 1414745144 | 35.0 | 283.0 | -132.0 | -114.0 | -430.0 | 132.0 | -86.0 | -38.0 | 326.0 | 2257 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
df1_complex = label_data('../../Footage/Footage.txt', 'complex')
df2_complex = label_data('../../Footage/Footage2.txt', 'complex')
df3_complex = label_data('../../Footage/Footage3.txt', 'complex')
df4_complex = label_data('../../Footage/Footage4.txt', 'complex')
df_concatenated = pd.concat([df1_complex, df2_complex, df3_complex, df4_complex])
print("Shape of first dataframe:", df1_complex.shape)
print("Shape of all combined dataframes:", df_concatenated.shape)
print("Printing dataframe...")
#print(df1_complex.head(10))
print(df_concatenated.head(10))Shape of first dataframe: (21645, 33)
Shape of all combined dataframes: (86580, 41)
Printing dataframe...
DISCARD DONE DONE, OUT OF WATER END SURF SESH \
ride_id
15692 1 0.0 NaN NaN NaN
2 0.0 NaN NaN NaN
3 0.0 NaN NaN NaN
4 0.0 NaN NaN NaN
5 0.0 NaN NaN NaN
6 0.0 NaN NaN NaN
7 0.0 NaN NaN NaN
8 0.0 NaN NaN NaN
9 0.0 NaN NaN NaN
10 0.0 NaN NaN NaN
FLIP BOARD RIGHT SIDE UP FLOATING IMU A1 IMU A2 IMU A3 \
ride_id
15692 1 NaN 0 493.0 48.0 110.0
2 NaN 0 513.0 89.0 62.0
3 NaN 0 494.0 92.0 80.0
4 NaN 0 421.0 205.0 -104.0
5 NaN 0 534.0 306.0 -32.0
6 NaN 0 455.0 149.0 -102.0
7 NaN 0 474.0 342.0 -219.0
8 NaN 0 363.0 323.0 -131.0
9 NaN 0 -21.0 510.0 -447.0
10 NaN 0 35.0 283.0 -132.0
IMU G1 IMU G2 IMU G3 IMU M1 IMU M2 IMU M3 LAY-DOWN NEW \
ride_id
15692 1 75.0 -124.0 -86.0 -309.0 209.0 39.0 0 0
2 34.0 -36.0 -92.0 -320.0 194.0 38.0 0 0
3 69.0 -63.0 -42.0 -329.0 189.0 49.0 0 0
4 192.0 -92.0 -37.0 -330.0 180.0 64.0 0 0
5 -421.0 -233.0 -229.0 -325.0 161.0 97.0 0 0
6 -355.0 -376.0 -397.0 -337.0 117.0 151.0 0 0
7 -234.0 -527.0 -465.0 -311.0 25.0 217.0 0 0
8 60.0 -662.0 -305.0 -238.0 -8.0 272.0 0 0
9 78.0 -643.0 -153.0 -159.0 -21.0 321.0 0 0
10 -114.0 -430.0 132.0 -86.0 -38.0 326.0 0 0
OFF-BOARD PADDLING PADDLING FOR A WAVE PADDLING FOR POSITION \
ride_id
15692 1 0 NaN 0 0.0
2 0 NaN 0 0.0
3 0 NaN 0 0.0
4 0 NaN 0 0.0
5 0 NaN 0 0.0
6 0 NaN 0 0.0
7 0 NaN 0 0.0
8 0 NaN 0 0.0
9 0 NaN 0 0.0
10 0 NaN 0 0.0
PADDLING INTO WAVES POP-UP PULL-BACK LEASH PUSH-OFF SIT-BACK \
ride_id
15692 1 0 0 0 0 0
2 0 0 0 0 0
3 0 0 0 0 0
4 0 0 0 0 0
5 0 0 0 0 0
6 0 0 0 0 0
7 0 0 0 0 0
8 0 0 0 0 0
9 0 0 0 0 0
10 0 0 0 0 0
SIT-UP STEP-OFF STOMACH SURFING SURFING SYNC \
ride_id
15692 1 0 0 NaN 0 0
2 0 0 NaN 0 0
3 0 0 NaN 0 0
4 0 0 NaN 0 0
5 0 0 NaN 0 0
6 0 0 NaN 0 0
7 0 0 NaN 0 0
8 0 0 NaN 0 0
9 0 0 NaN 0 0
10 0 0 NaN 0 0
TURNING TO SURFER'S LEFT TURNING TO SURFER'S RIGHT TURTLE ROLL \
ride_id
15692 1 0 0 0.0
2 0 0 0.0
3 0 0 0.0
4 0 0 0.0
5 0 0 0.0
6 0 0 0.0
7 0 0 0.0
8 0 0 0.0
9 0 0 0.0
10 0 0 0.0
Time TimeDelta UNKNOWN UTC \
ride_id
15692 1 1414742887 0 NaN 2018-11-09T19:16:03.8090+00:00
2 1414743138 251 NaN 2018-11-09T19:16:04.0610+00:00
3 1414743387 500 NaN 2018-11-09T19:16:04.3120+00:00
4 1414743639 752 NaN 2018-11-09T19:16:04.5650+00:00
5 1414743889 1002 NaN 2018-11-09T19:16:04.8170+00:00
6 1414744139 1252 NaN 2018-11-09T19:16:05.0680+00:00
7 1414744390 1503 NaN 2018-11-09T19:16:05.3210+00:00
8 1414744641 1754 NaN 2018-11-09T19:16:05.5730+00:00
9 1414744892 2005 NaN 2018-11-09T19:16:05.8260+00:00
10 1414745144 2257 NaN 2018-11-09T19:16:06.0790+00:00
WALKING IN WATER WALKING OUT OF WATER WIPE-OUT
ride_id
15692 1 0 NaN 0
2 0 NaN 0
3 0 NaN 0
4 0 NaN 0
5 0 NaN 0
6 0 NaN 0
7 0 NaN 0
8 0 NaN 0
9 0 NaN 0
10 0 NaN 0
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:6: FutureWarning:
Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.
To accept the future behavior, pass 'sort=False'.
To retain the current behavior and silence the warning, pass 'sort=True'.
#correct IMU data
#make a deep copy of motion_df_labelled
df_converted = motion_df_complex.copy(deep = 'true')
#for rows in df_corrected
for row in range(0, df_converted.shape[0]):
#convert acceleromters (new: m/s^2)
df_converted.iloc[row, df_converted.columns.get_loc('IMU A1')] *= -0.019141 #forwards/backwards
df_converted.iloc[row, df_converted.columns.get_loc('IMU A2')] *= 0.019141 #upside down/right side up
df_converted.iloc[row, df_converted.columns.get_loc('IMU A3')] *= 0.019141 #sideways: negative = left, positive = right
#convert gyroscopes (new: deg/s)
df_converted.iloc[row, df_converted.columns.get_loc('IMU G1')] /= 8.2 #roll
df_converted.iloc[row, df_converted.columns.get_loc('IMU G2')] /= 8.2 #yaw
df_converted.iloc[row, df_converted.columns.get_loc('IMU G3')] /= 8.2 #pitch (flipping forwards/backwards)
motion_df_complex.head(10)
<style scoped>
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
</style>
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| UTC | Time | IMU A1 | IMU A2 | IMU A3 | IMU G1 | IMU G2 | IMU G3 | IMU M1 | IMU M2 | IMU M3 | TimeDelta | SYNC | FLIP BOARD RIGHT SIDE UP | WALKING IN WATER | PUSH-OFF | PADDLING INTO WAVES | SIT-UP | FLOATING | TURNING TO SURFER'S LEFT | LAY-DOWN | PADDLING FOR A WAVE | POP-UP | SURFING | STEP-OFF | TURNING TO SURFER'S RIGHT | SIT-BACK | OFF-BOARD | PADDLING | WIPE-OUT | PULL-BACK LEASH | PADDLING FOR POSITION | NEW | DISCARD | DONE, OUT OF WATER | WALKING OUT OF WATER | ||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ride_id | |||||||||||||||||||||||||||||||||||||
| 15692 | 1 | 2018-11-09T19:16:03.8090+00:00 | 1414742887 | 493.0 | 48.0 | 110.0 | 75.0 | -124.0 | -86.0 | -309.0 | 209.0 | 39.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 2 | 2018-11-09T19:16:04.0610+00:00 | 1414743138 | 513.0 | 89.0 | 62.0 | 34.0 | -36.0 | -92.0 | -320.0 | 194.0 | 38.0 | 251 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 3 | 2018-11-09T19:16:04.3120+00:00 | 1414743387 | 494.0 | 92.0 | 80.0 | 69.0 | -63.0 | -42.0 | -329.0 | 189.0 | 49.0 | 500 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 4 | 2018-11-09T19:16:04.5650+00:00 | 1414743639 | 421.0 | 205.0 | -104.0 | 192.0 | -92.0 | -37.0 | -330.0 | 180.0 | 64.0 | 752 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 5 | 2018-11-09T19:16:04.8170+00:00 | 1414743889 | 534.0 | 306.0 | -32.0 | -421.0 | -233.0 | -229.0 | -325.0 | 161.0 | 97.0 | 1002 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 6 | 2018-11-09T19:16:05.0680+00:00 | 1414744139 | 455.0 | 149.0 | -102.0 | -355.0 | -376.0 | -397.0 | -337.0 | 117.0 | 151.0 | 1252 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 7 | 2018-11-09T19:16:05.3210+00:00 | 1414744390 | 474.0 | 342.0 | -219.0 | -234.0 | -527.0 | -465.0 | -311.0 | 25.0 | 217.0 | 1503 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 8 | 2018-11-09T19:16:05.5730+00:00 | 1414744641 | 363.0 | 323.0 | -131.0 | 60.0 | -662.0 | -305.0 | -238.0 | -8.0 | 272.0 | 1754 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 9 | 2018-11-09T19:16:05.8260+00:00 | 1414744892 | -21.0 | 510.0 | -447.0 | 78.0 | -643.0 | -153.0 | -159.0 | -21.0 | 321.0 | 2005 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 10 | 2018-11-09T19:16:06.0790+00:00 | 1414745144 | 35.0 | 283.0 | -132.0 | -114.0 | -430.0 | 132.0 | -86.0 | -38.0 | 326.0 | 2257 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
df_converted.head(10)
<style scoped>
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
</style>
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| UTC | Time | IMU A1 | IMU A2 | IMU A3 | IMU G1 | IMU G2 | IMU G3 | IMU M1 | IMU M2 | IMU M3 | TimeDelta | SYNC | FLIP BOARD RIGHT SIDE UP | WALKING IN WATER | PUSH-OFF | PADDLING INTO WAVES | SIT-UP | FLOATING | TURNING TO SURFER'S LEFT | LAY-DOWN | PADDLING FOR A WAVE | POP-UP | SURFING | STEP-OFF | TURNING TO SURFER'S RIGHT | SIT-BACK | OFF-BOARD | PADDLING | WIPE-OUT | PULL-BACK LEASH | PADDLING FOR POSITION | NEW | DISCARD | DONE, OUT OF WATER | WALKING OUT OF WATER | ||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| ride_id | |||||||||||||||||||||||||||||||||||||
| 15692 | 1 | 2018-11-09T19:16:03.8090+00:00 | 1414742887 | -9.436513 | 0.918768 | 2.105510 | 9.146341 | -15.121951 | -10.487805 | -309.0 | 209.0 | 39.0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 2 | 2018-11-09T19:16:04.0610+00:00 | 1414743138 | -9.819333 | 1.703549 | 1.186742 | 4.146341 | -4.390244 | -11.219512 | -320.0 | 194.0 | 38.0 | 251 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 3 | 2018-11-09T19:16:04.3120+00:00 | 1414743387 | -9.455654 | 1.760972 | 1.531280 | 8.414634 | -7.682927 | -5.121951 | -329.0 | 189.0 | 49.0 | 500 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 4 | 2018-11-09T19:16:04.5650+00:00 | 1414743639 | -8.058361 | 3.923905 | -1.990664 | 23.414634 | -11.219512 | -4.512195 | -330.0 | 180.0 | 64.0 | 752 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 5 | 2018-11-09T19:16:04.8170+00:00 | 1414743889 | -10.221294 | 5.857146 | -0.612512 | -51.341463 | -28.414634 | -27.926829 | -325.0 | 161.0 | 97.0 | 1002 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 6 | 2018-11-09T19:16:05.0680+00:00 | 1414744139 | -8.709155 | 2.852009 | -1.952382 | -43.292683 | -45.853659 | -48.414634 | -337.0 | 117.0 | 151.0 | 1252 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 7 | 2018-11-09T19:16:05.3210+00:00 | 1414744390 | -9.072834 | 6.546222 | -4.191879 | -28.536585 | -64.268293 | -56.707317 | -311.0 | 25.0 | 217.0 | 1503 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 8 | 2018-11-09T19:16:05.5730+00:00 | 1414744641 | -6.948183 | 6.182543 | -2.507471 | 7.317073 | -80.731707 | -37.195122 | -238.0 | -8.0 | 272.0 | 1754 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 9 | 2018-11-09T19:16:05.8260+00:00 | 1414744892 | 0.401961 | 9.761910 | -8.556027 | 9.512195 | -78.414634 | -18.658537 | -159.0 | -21.0 | 321.0 | 2005 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | |
| 10 | 2018-11-09T19:16:06.0790+00:00 | 1414745144 | -0.669935 | 5.416903 | -2.526612 | -13.902439 | -52.439024 | 16.097561 | -86.0 | -38.0 | 326.0 | 2257 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
list(df_converted)['UTC',
'Time',
'IMU A1',
'IMU A2',
'IMU A3',
'IMU G1',
'IMU G2',
'IMU G3',
'IMU M1',
'IMU M2',
'IMU M3',
'TimeDelta',
'SYNC',
'FLIP BOARD RIGHT SIDE UP',
'WALKING IN WATER',
'PUSH-OFF',
'PADDLING INTO WAVES',
'SIT-UP',
'FLOATING',
"TURNING TO SURFER'S LEFT",
'LAY-DOWN',
'PADDLING FOR A WAVE',
'POP-UP',
'SURFING',
'STEP-OFF',
"TURNING TO SURFER'S RIGHT",
'SIT-BACK',
'OFF-BOARD',
'PADDLING',
'WIPE-OUT',
'PULL-BACK LEASH',
'PADDLING FOR POSITION',
'NEW',
'DISCARD',
'DONE, OUT OF WATER',
'WALKING OUT OF WATER']
## Drop data columns that we don't care about predicting/visualizing:
df_converted = df_converted.drop(columns=["FLIP BOARD RIGHT SIDE UP", "NEW", "DONE, OUT OF WATER"])
#df_converted = df_converted.drop(columns!=["SURFING, FLOATING, PADDLING INTO WAVES, PADDLING FOR A WAVE, PADDLING FOR POSITION, PADDLING"])import matplotlib.pyplot as plt
%matplotlib inline
plt.rcParams['figure.figsize'] = [50, 10]
#define a function that plots a column of dataf in relation to time. color coded to match labels in dataf
#requires that:
#dataf has a 'TimeDelta' column
#labels: walking, surfing, floating, paddling
def createPlot (dataf, column):
#create new data frame to be plotted
#Only consider columns after Velocity
dfPlot = pd.DataFrame(columns = ['TIME'] + list(dataf)[list(dataf).index('TimeDelta') + 1:], dtype = float)
#add timedelta column from dataf to dfPlot
dfPlot['TIME'] = dataf['TimeDelta']
#get the index of the column to be graphed
columnInd = dataf.columns.get_loc(column)
#for each row in dfPlot (number of IMU readings)
for row in range(0, dfPlot.shape[0]):
#for the indexes of the label columns in dfPlot
for col in range(1, dfPlot.shape[1]):
#if a label in the row is 1 in dataf
if dataf.iloc[row, dataf.columns.get_loc(dfPlot.columns[col])] == 1:
#add the sensors value to the corresponding column in dfPlot
dfPlot.iloc[row, dfPlot.columns.get_loc(dfPlot.columns[col])] = dataf.iloc[row, columnInd]
#dfPlot.iloc[row, dfPlot.columns.get]
#Set up colormap so that we don't see a repeat in color when graphing
#plt.gca().set_prop_cycle('color',plt.cm.plasma(np.linspace(0,1,dfPlot.shape[1])))
plt.gca().set_prop_cycle('color',plt.cm.tab20(np.linspace(0,1,dfPlot.shape[1])))
for col in range (1, dfPlot.shape[1]):
plt.plot(dfPlot['TIME'], dfPlot[list(dfPlot)[col]])
plt.gca().legend(loc = 'lower left')
plt.title(column)
plt.xlabel("Time")
plt.ylabel("IMU Data")
#file_name = column
#pdf_string = '.jpg'
#file_name += pdf_string
#plt.savefig(file_name)
plt.show()
return#Need to clear kernel and then only run all above so that it plots on axes directly below, rather than on another plot
print("Creating Plots...")
createPlot(df_converted,'IMU A1')
createPlot(df_converted,'IMU A2')
createPlot(df_converted,'IMU A3')
createPlot(df_converted,'IMU G1')
createPlot(df_converted,'IMU G2')
createPlot(df_converted,'IMU G3')
createPlot(df_converted,'IMU M1')
createPlot(df_converted,'IMU M2')
createPlot(df_converted,'IMU M3')
print("Done")Creating Plots...
Done
# Define function to choose floating data from graph and plot
def selectData (dataf, column):
#create new data frame to be plotted
#Only consider columns after Velocity
dfPlot = pd.DataFrame(columns = ['TIME'] + list(dataf)[list(dataf).index('SIT-UP') + 1: list(dataf).index("FLOATING") + 1], dtype = float)
#add timedelta column from dataf to dfPlot
dfPlot['TIME'] = dataf['TimeDelta']
#get the index of the column to be graphed
columnInd = dataf.columns.get_loc(column)
#for each row in dfPlot (number of IMU readings)
for row in range(0, dfPlot.shape[0]):
#for the indexes of the label columns in dfPlot
for col in range(1, dfPlot.shape[1]):
#if a label in the row is 1 in dataf
if dataf.iloc[row, dataf.columns.get_loc(dfPlot.columns[col])] == 1:
#add the sensors value to the corresponding column in dfPlot
dfPlot.iloc[row, dfPlot.columns.get_loc(dfPlot.columns[col])] = dataf.iloc[row, columnInd]
#dfPlot.iloc[row, dfPlot.columns.get]
# If 0, print time stamps between rows 1000000 and 1280000
else:
if 1016600 <= dataf['TimeDelta'][row] <= 1300000:
print("Selected Region: 1016600 ms to 1300000 ms")
for col in range (1, dfPlot.shape[1]):
plt.plot(dfPlot['TIME'], dfPlot[list(dfPlot)[col]], color='g')
plt.gca().legend(loc = 'lower left')
plt.title(column)
plt.xlabel("Time")
plt.ylabel("IMU Data")
plt.xlim(left=1016600, right=1300000)
#file_name = column
#pdf_string = '.jpg'
#file_name += pdf_string
#plt.savefig(file_name)
plt.show()
return# Select Floating Data to analyze
selectData(df_converted,'IMU M1')
selectData(df_converted,'IMU M2')
selectData(df_converted,'IMU M3')Selected Region: 1016600 ms to 1300000 ms
Selected Region: 1016600 ms to 1300000 ms
Selected Region: 1016600 ms to 1300000 ms
# To store time elapsed between each measurement, and time offset from 0s
time_e_list = []
time_o_list = []
#Remove all nan instances in time:
time_array_nans = np.array(dropped_motion_df.loc[:,"Time"], dtype=float)
time_array = []
imuA1_array_nans = np.array(dropped_motion_df.loc[:,"IMU A1"], dtype=float)
imu_array_A1 = []
imuA2_array_nans = np.array(dropped_motion_df.loc[:,"IMU A2"], dtype=float)
imu_array_A2 = []
imuA3_array_nans = np.array(dropped_motion_df.loc[:,"IMU A3"], dtype=float)
imu_array_A3 = []
imuG1_array_nans = np.array(dropped_motion_df.loc[:,"IMU G1"], dtype=float)
imu_array_G1 = []
imuG2_array_nans = np.array(dropped_motion_df.loc[:,"IMU G2"], dtype=float)
imu_array_G2 = []
imuG3_array_nans = np.array(dropped_motion_df.loc[:,"IMU G3"], dtype=float)
imu_array_G3 = []
imuM1_array_nans = np.array(dropped_motion_df.loc[:,"IMU M1"], dtype=float)
imu_array_M1 = []
imuM2_array_nans = np.array(dropped_motion_df.loc[:,"IMU M2"], dtype=float)
imu_array_M2 = []
imuM3_array_nans = np.array(dropped_motion_df.loc[:,"IMU M3"], dtype=float)
imu_array_M3 = []
#Get all the times and imus where time, imu a1, imu a2, and imu a3 are NOT nan values:
for t,x,y,z,a,b,c,d,e,f in zip(time_array_nans, imuA1_array_nans, imuA2_array_nans, imuA3_array_nans, imuG1_array_nans,
imuG2_array_nans, imuG3_array_nans, imuM1_array_nans, imuM2_array_nans, imuM3_array_nans):
if (np.isnan(t)==0 and np.isnan(x)==0 and np.isnan(y)==0 and np.isnan(z)==0):
time_array.append(t)
imu_array_A1.append(x)
imu_array_A2.append(y)
imu_array_A3.append(z)
imu_array_G1.append(a)
imu_array_G2.append(b)
imu_array_G3.append(c)
imu_array_M1.append(d)
imu_array_M2.append(e)
imu_array_M3.append(f)
#for x in time_array:
# print(x)
start_time = time_array[0]
time_len = len(time_array)
i = 0
while (i < time_len - 1):
prev = time_array[i]
after = time_array[i+1]
offset = after - prev
#if (np.isnan(offset)==0):
time_o_list.append(offset)
elapsed = time_array[i] - start_time
#if (np.isnan(elapsed)==0):
time_e_list.append(elapsed)
i = i + 1
##Check to make sure there are no "nan" values:
i = 0
while (i < len(time_o_list)):
if (np.isnan(time_o_list[i])):
print("Error! Value at index: ", i, " is nan")
i = i + 1
#Drop the last value from each of the imu lists to make it match the time list.
del(imu_array_A1[-1])
del(imu_array_A2[-1])
del(imu_array_A3[-1])
del(imu_array_G1[-1])
del(imu_array_G2[-1])
del(imu_array_G3[-1])
del(imu_array_M1[-1])
del(imu_array_M2[-1])
del(imu_array_M3[-1])
print(len(time_e_list))
print(len(time_o_list))
print(len(imu_array_A1))
print(len(imu_array_A2))
print(len(imu_array_A3))
print(len(imu_array_G1))
print(len(imu_array_G2))
print(len(imu_array_G3))
print(len(imu_array_M1))
print(len(imu_array_M2))
print(len(imu_array_M3))
CheckLength = len(time_e_list) + len(time_o_list) + len(imu_array_A1) + len(imu_array_A2) + len(imu_array_A3) + len(imu_array_G1) + len(imu_array_G2) + len(imu_array_G3) + len(imu_array_M1)+ len(imu_array_M2) + len(imu_array_M3)
if CheckLength//11 == len(time_e_list):
print("All columns are matching!")21644
21644
21644
21644
21644
21644
21644
21644
21644
21644
21644
All columns are matching!
#Raw acceleration constant 512 = 1g (accelerometer's measured force due to gravity)
g_const = 512
#Approximate measurement for gravity:
gravity = 9.80665
# Correct the IMU Acceleration columns into units of meters
def convert_acc_units(acc_array):
ret_array = []
for a in acc_array:
#Acceleration is now in m/s^2, need to subtract gravity from vertical axis. (??)
new_a = a / g_const * gravity
ret_array.append(new_a)
return ret_array
imu1_array_calib = convert_acc_units(imu_array_A1) #new units in m/s^2
imu2_array_calib = convert_acc_units(imu_array_A2) #new units in m/s^2
imu3_array_calib = convert_acc_units(imu_array_A3) #new units in m/s^2
# To check:
#for x,y in zip(imu2_array, imu_array_A2):
# print(x,y)
def convert_time_units(time_array):
ret_array = []
for t in time_array:
new_t = t * (10**(-3)) #converting units in milliseconds to seconds
ret_array.append(new_t)
return ret_array
# To check:
# for t in time_e_array:
# print(t)
print("Done!")Done!
print("Graph of X Acceleration vs. Time")
plt.plot(time_e_array, imu1_array_calib)
plt.xlabel("Time (s)")
plt.ylabel("Acceleration-X (m/s^2)")
plt.show()Graph of X Acceleration vs. Time
# Offset variables help in recentering the magnetic data in order to define direction and use trig functions
M1_offset_var = 219.786
M2_offset_var = 180
M3_offset_var = 280
def calibrate_magn_data(magn_array, offset_value):
ret_array = []
for m in magn_array:
new_m = m - offset_value
ret_array.append(new_m)
return ret_array
imuM1_array_calib = calibrate_magn_data(imu_array_M1, M1_offset_var)
imuM2_array_calib = calibrate_magn_data(imu_array_M2, M2_offset_var)
imuM3_array_calib = calibrate_magn_data(imu_array_M3, M3_offset_var)
# Check
print(len(imuM1_array_calib))
print("Done.")21644
Done.
lower_time = 1016600
upper_time = 1300000
# Create new arrays for Magnetometer values
imuM1_array = []
imuM2_array = []
imuM3_array = []
# Create new arrays for accelerometer values
imu1_array = []
imu2_array = []
imu3_array = []
new_time_e_list = []
# iterate through time_e_array
for i in range(0, len(time_e_list)):
if lower_time <= time_e_list[i] <= upper_time:
# Set new time offset
new_time_e_list.append(time_e_list[i] - 1016600)
# Set 3-D IMU values
imuM1_array.append(imuM1_array_calib[i])
imuM2_array.append(imuM2_array_calib[i])
imuM3_array.append(imuM3_array_calib[i])
imu1_array.append(imu1_array_calib[i])
imu2_array.append(imu2_array_calib[i])
imu3_array.append(imu3_array_calib[i])
# Convert time units to seconds
time_e_array = convert_time_units(new_time_e_list) #new units in seconds
# Check if all array lengths match
CheckLength = len(time_e_array) + len(imu1_array) + len(imu2_array) + len(imu3_array) + len(imuM1_array) + len(imuM2_array) + len(imuM3_array)
if CheckLength == 7*len(new_time_e_list):
print("All columns are matching!")
print("Data Selected!")All columns are matching!
Data Selected!
# Create N x 3 arrays for functions that need them later on, such as Scikit Kinematics
magn_height = len(imuM1_array)
acc_height = len(imu1_array)
acc_array = np.zeros(shape=(acc_height,3))
magn_array = np.zeros(shape=(magn_height,3))
print("For Accelerometer: ")
for x in range(len(acc_array)):
acc_array[x,0] = imu1_array[x]
acc_array[x,1] = imu2_array[x]
acc_array[x,2] = imu3_array[x]
print(acc_array)
print("\nFor Magnetometer: ")
for x in range(len(magn_array)):
magn_array[x,0] = imuM1_array[x]
magn_array[x,1] = imuM2_array[x]
magn_array[x,2] = imuM3_array[x]
print(magn_array)
print("Done.")For Accelerometer:
[[ 0.68953008 9.42357773 2.43250889]
[ 0.61291562 10.32379756 2.27927998]
[ 0.84275898 9.84495723 2.03028301]
...
[ 0.78529814 9.13627354 1.81959326]
[-0.26815059 9.69172832 2.29843359]
[ 1.09175596 9.44273135 2.77727393]]
For Magnetometer:
[[-294.786 -361. -107. ]
[-313.786 -360. -94. ]
[-317.786 -352. -88. ]
...
[-227.786 -312. -350. ]
[-247.786 -316. -332. ]
[-246.786 -321. -343. ]]
Done.
Orientation from here onwards will be from the board/surfers reference frame (yaw left = turning left)
x = -IMU1, y = -IMU3, z = -IMU2
# The new array for board reference frame will have the IMUs in columns according to X,Y,Z directions
print("For Accelerometer:")
board_acc = acc_array.copy() # Reassign to the correct axes as stated above
temp_x_acc = board_acc[:,0] * (-1)
temp_y_acc = board_acc[:,1] * (-1)
temp_z_acc = board_acc[:,2] * (-1)
board_acc[:,0] = temp_x_acc # X acceleration
board_acc[:,1] = temp_y_acc # Y acceleration
board_acc[:,2] = temp_z_acc # Z acceleration
print(board_acc)
print("\nFor Magnetometer:")
board_magn = magn_array.copy()
temp_x_magn = board_magn[:,0] * (-1)
temp_y_magn = board_magn[:,1] * (-1)
temp_z_magn = board_magn[:,2] * (-1)
board_magn[:,0] = temp_x_magn
board_magn[:,1] = temp_y_magn
board_magn[:,2] = temp_z_magn
print(board_magn)
print("Done.")For Accelerometer:
[[ -0.68953008 -9.42357773 -2.43250889]
[ -0.61291562 -10.32379756 -2.27927998]
[ -0.84275898 -9.84495723 -2.03028301]
...
[ -0.78529814 -9.13627354 -1.81959326]
[ 0.26815059 -9.69172832 -2.29843359]
[ -1.09175596 -9.44273135 -2.77727393]]
For Magnetometer:
[[294.786 361. 107. ]
[313.786 360. 94. ]
[317.786 352. 88. ]
...
[227.786 312. 350. ]
[247.786 316. 332. ]
[246.786 321. 343. ]]
Done.
# Azimuth and Altitude LEGEND:
# Altitude is the angle between the ground and the vector
# Azimuth is the angle going clockwise from 0 deg North:
# N - 0/360deg, E - 90deg, S - 180deg, W - 270deg
# This will get complicated (ie make cases or lots of if statements) when rotations about the heading become more prevalent
def azimuth(x,y,z):
real_y = y * (-1) # This is to account for y
return (180/math.pi * math.atan2(real_y,x)) % 360
def altitude(x,y,z):
h = math.hypot(y, x)
return 180/math.pi * math.atan2(z,h)
def printAltAzi(alt, azi):
print ("Alt:", alt, "\n", "Azi:",azi,"\n")
# These values are uncorrected values: still need to add or subtract 'declination'
# (for AziMuth) and 'inclination' (for Altitude) correction values for geographical location
heading_altitude = board_magn[:,0].copy()
heading_azimuth = board_magn[:,0].copy()
i = 0 #iterator
#for i in range(len(M1_no_out)):
while i < len(heading_altitude):
factor = 0;
# Use acceleration values to calibrate magnetometer
if (board_acc[i, 0] >= 0):
factor = 1
else:
factor = -1
heading_altitude[i] = altitude(board_magn[i,0], board_magn[i,1], board_magn[i,2])
heading_azimuth[i] = azimuth(board_magn[i,0], board_magn[i,1], board_magn[i,2])
heading_azimuth[i] = heading_azimuth[i] * factor
#printAltAzi(heading_altitude[i],heading_azimuth[i])
i += 1
#for t in range(len(time_e_array)):
#printAltAzi(heading_altitude[t], heading_azimuth[t])# Fixing random state for reproducibility
heading_alt_plot = plt.figure(figsize=(10,5))
alt_plot = heading_alt_plot.add_subplot(111)
alt_plot.plot(time_e_array, heading_altitude)
alt_plot.set_title("Altitude vs. Time")
alt_plot.set_xlabel("Time Elapsed [sec]")
alt_plot.set_ylabel("Altitude [deg]")
plt.show()
np.random.seed(19680801)
# Compute areas and colors
r = [i for i in range(0, len(board_magn))]
theta = heading_azimuth/360 * 2 * np.pi
area = 1
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111, projection='polar')
c = ax.scatter(theta, r, c='red', s=area, cmap='hsv', alpha=1)
# Compute areas and colors
alt_r = [i for i in range(0, len(board_magn))]
alt_theta = heading_altitude/360 * 2 * np.pi
alt_area = 1
alt_colors = theta
fig2 = plt.figure(figsize=(10,10))
ax2 = fig2.add_subplot(111, projection='polar')
c2 = ax2.scatter(alt_theta, alt_r, c='blue', s=alt_area, cmap='hsv', alpha=1)
# Plot first 10 directional changes
magnfig = plt.figure(figsize=(20,20))
magnaxi = magnfig.add_subplot(111, projection='3d')
magnaxi.scatter(board_magn[0,0], board_magn[0,1], board_magn[0,2], c='black', s=300, marker = "<")
magnaxi.scatter(board_magn[1:,0], board_magn[1:,1], board_magn[1:,2], c='black', s=10, marker = "o")
magnaxi.plot(board_magn[:,0], board_magn[:,1], board_magn[:,2], color='green')
magnaxi.set_xlabel("X Axis ->", fontsize=30)
magnaxi.set_ylabel("Y Axis ->", fontsize=30)
magnaxi.set_zlabel("Z Axis ->", fontsize=30)
plt.show()
# Plot Smartfin data
heading_azi_plot = plt.figure(figsize=(10,5))
azi_plot = heading_azi_plot.add_subplot(111)
azi_plot.plot(time_e_array, heading_azimuth)
azi_plot.set_title("Azimuth vs. Time")
azi_plot.set_xlabel("Time Elapsed[sec]")
azi_plot.set_ylabel("Azimuth [deg]")
print()
# Get buoy data from CDIP website
# Link: https://cdip.ucsd.edu/themes/?d2=p70:s:201&zoom=auto&pub_set=public®ions=california_south&tz=UTC&units=standard
# Calculate average heading using my own graph
total_sum = 0
for i in heading_azimuth:
total_sum += i
my_average = total_sum/len(heading_azimuth)
expected_average = 282.41
print("My calculated average heading (deg) is:")
print(my_average)
heading_average = 360 + my_average
print("\nWhich is a [SW] heading of: ")
print(heading_average)
#print("\nThe expected heading (deg) is:")
#print(expected_average)
# Calculate error in findings
#error_percent = (my_average - expected_average) / expected_average
#print("\nThe percentage error in my model is:")
#print(error_percent)My calculated average heading (deg) is:
-164.47835236359094
Which is a [SW] heading of:
195.52164763640906