Feature: Multiple Playlist Deduplication

.gitignore

@@ -22,4 +22,15 @@
 # debug
 npm-debug.log*
 yarn-debug.log*
-yarn-error.log*
+yarn-error.log*
+
+#vscode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/

components/duplicateTrackListItem.tsx

@@ -4,17 +4,18 @@ import Badge from './badge';
 
 export const DuplicateTrackListItem = ({
   reason,
+  firstPlaylistName,
   trackName,
   trackArtistName,
 }) => {
   const { t, i18n } = useTranslation();
   return (
     <li>
       {reason === 'same-id' && (
-        <Badge>{t('result.duplicate.reason-same-id')}</Badge>
+        <Badge>{t('result.duplicate.reason-same-id')} {firstPlaylistName ? " from " + firstPlaylistName : ""} </Badge>
       )}
       {reason === 'same-name-artist' && (
-        <Badge>{t('result.duplicate.reason-same-data')}</Badge>
+        <Badge>{t('result.duplicate.reason-same-data')} {firstPlaylistName ? " from " + firstPlaylistName : ""}</Badge>
       )}
       <Trans i18nKey="result.duplicate.track">
         <span>{{ trackName }}</span> <span className="gray">by</span>{' '}

components/intro.tsx

@@ -34,6 +34,8 @@ const Intro = props => {
             {t('home.login-button')}
           </button>
         </p>
+        <label>de-duplicate across playlists?</label>
+        <input type="checkbox" onChange={props.onMultiPlaylistToggle}></input>
       </div>
       <style jsx>
         {`

components/main.tsx

@@ -53,6 +53,7 @@ export default class Main extends React.Component<{
   api: any;
   user: SpotifyUserType;
   accessToken: string;
+  multiPlaylistDedup: boolean;
 }> {
   state: StateType = {
     toProcess: null,
@@ -68,7 +69,7 @@ export default class Main extends React.Component<{
     process.on('updateState', (state) => {
       this.setState(state);
     });
-    process.process(this.props.api, this.props.user);
+    process.process(this.props.api, this.props.user, this.props.multiPlaylistDedup);
 
     const hasUsedSpotifyTop = async () => {
       const res = await fetch(`https://spotify-top.com/api/profile`, {
@@ -84,8 +85,8 @@ export default class Main extends React.Component<{
         .then((result) => {
           this.setState({ hasUsedSpotifyTop: result === true });
         })
-        .catch((e) => {});
-    } catch (e) {}
+        .catch((e) => { });
+    } catch (e) { }
   }
 
   removeDuplicates = (playlist: PlaylistModel) => {
@@ -167,9 +168,9 @@ export default class Main extends React.Component<{
       this.state.playlists.length === 0
         ? 0
         : this.state.playlists.reduce(
-            (prev, current) => prev + current.duplicates.length,
-            0
-          ) + this.state.savedTracks.duplicates.length;
+          (prev, current) => prev + current.duplicates.length,
+          0
+        ) + this.state.savedTracks.duplicates.length;
 
     return (
       <div>
@@ -246,62 +247,63 @@ export default class Main extends React.Component<{
         <ul className="playlists-list">
           {(this.state.savedTracks.duplicates.length ||
             this.state.savedTracks.status) && (
-            <li className="playlists-list-item media">
-              <div className="img">
-                <img
-                  width="100"
-                  height="100"
-                  className="playlists-list-item__img"
-                  src={'./placeholder.png'}
-                />
-              </div>
-              <div className="bd">
-                <span className="playlists-list-item__name">
-                  <Translation>{(t) => t('process.saved.title')}</Translation>
-                </span>
-                {this.state.savedTracks.status && (
-                  <Badge>
-                    <Translation>
-                      {(t) => t(this.state.savedTracks.status)}
-                    </Translation>
-                  </Badge>
-                )}
-                {this.state.savedTracks.duplicates.length != 0 && (
-                  <span>
-                    <span>
+              <li className="playlists-list-item media">
+                <div className="img">
+                  <img
+                    width="100"
+                    height="100"
+                    className="playlists-list-item__img"
+                    src={'./placeholder.png'}
+                  />
+                </div>
+                <div className="bd">
+                  <span className="playlists-list-item__name">
+                    <Translation>{(t) => t('process.saved.title')}</Translation>
+                  </span>
+                  {this.state.savedTracks.status && (
+                    <Badge>
                       <Translation>
-                        {(t) =>
-                          t('process.saved.duplicates', {
-                            count: this.state.savedTracks.duplicates.length,
-                          })
-                        }
+                        {(t) => t(this.state.savedTracks.status)}
                       </Translation>
+                    </Badge>
+                  )}
+                  {this.state.savedTracks.duplicates.length != 0 && (
+                    <span>
+                      <span>
+                        <Translation>
+                          {(t) =>
+                            t('process.saved.duplicates', {
+                              count: this.state.savedTracks.duplicates.length,
+                            })
+                          }
+                        </Translation>
+                      </span>
+                      <button
+                        className="btn btn-primary btn-sm playlist-list-item__btn"
+                        onClick={() => this.removeDuplicatesInSavedTracks()}
+                      >
+                        <Translation>
+                          {(t) => t('process.saved.remove-button')}
+                        </Translation>
+                      </button>
+                      <DuplicateTrackList>
+                        {this.state.savedTracks.duplicates.map(
+                          (duplicate, index) => (
+                            <DuplicateTrackListItem
+                              key={index}
+                              reason={duplicate.reason}
+                              trackName={duplicate.track.name}
+                              trackArtistName={duplicate.track.artists[0].name}
+                              firstPlaylistName={null}
+                            />
+                          )
+                        )}
+                      </DuplicateTrackList>
                     </span>
-                    <button
-                      className="btn btn-primary btn-sm playlist-list-item__btn"
-                      onClick={() => this.removeDuplicatesInSavedTracks()}
-                    >
-                      <Translation>
-                        {(t) => t('process.saved.remove-button')}
-                      </Translation>
-                    </button>
-                    <DuplicateTrackList>
-                      {this.state.savedTracks.duplicates.map(
-                        (duplicate, index) => (
-                          <DuplicateTrackListItem
-                            key={index}
-                            reason={duplicate.reason}
-                            trackName={duplicate.track.name}
-                            trackArtistName={duplicate.track.artists[0].name}
-                          />
-                        )
-                      )}
-                    </DuplicateTrackList>
-                  </span>
-                )}
-              </div>
-            </li>
-          )}
+                  )}
+                </div>
+              </li>
+            )}
           {this.state.playlists
             .filter((p) => p.duplicates.length || p.status != '')
             .map((playlist: PlaylistModel, index) => (
@@ -353,6 +355,7 @@ export default class Main extends React.Component<{
                             reason={duplicate.reason}
                             trackName={duplicate.track.name}
                             trackArtistName={duplicate.track.artists[0].name}
+                            firstPlaylistName={duplicate.firstPlaylist ? duplicate.firstPlaylist.name : null}
                           />
                         ))}
                       </DuplicateTrackList>

components/pages/index.js

@@ -27,9 +27,8 @@ const MetaHead = () => {
       <meta name="viewport" content="width=device-width" />
       <link
         rel="canonical"
-        href={`https://spotify-dedup.com/${
-          i18n.language === 'en' ? '' : i18n.language + '/'
-        }`}
+        href={`https://spotify-dedup.com/${i18n.language === 'en' ? '' : i18n.language + '/'
+          }`}
       />
       {AvailableLanguages.filter((language) => language !== i18n.language).map(
         (language) => (
@@ -56,9 +55,14 @@ export default class Index extends React.Component {
   state = {
     isLoggedIn: false,
     user: null,
+    multiPlaylistDedup: false
   };
   api = null;
 
+  handleMultiPlaylistToggle = () => {
+    this.setState({ multiPlaylistDedup: !this.state.multiPlaylistDedup });
+  }
+
   handleLoginClick = async () => {
     const accessToken = await OAuthManager.obtainToken({
       scopes: [
@@ -100,9 +104,10 @@ export default class Index extends React.Component {
             api={Index.api}
             user={this.state.user}
             accessToken={this.state.accessToken}
+            multiPlaylistDedup={this.state.multiPlaylistDedup}
           />
         ) : (
-          <Intro onLoginClick={this.handleLoginClick} />
+          <Intro onLoginClick={this.handleLoginClick} onMultiPlaylistToggle={this.handleMultiPlaylistToggle} />
         )}
         {this.state.isLoggedIn
           ? null

dedup/deduplicator.ts

@@ -60,7 +60,130 @@ class BaseDeduplicator {
   }
 }
 
+export class MultiPlaylistDeduplicator {
+  static findDuplicatedTracks(tracks: Array<SpotifyPlaylistTrackType>) {
+    const seenIds: { [key: string]: boolean } = {};
+    // the earliest date when a track was inserted to any of user's playlists
+    const seenNameAndArtistDate: { [key: string]: Date } = {};
+    const seenNameAndArtist: { [key: string]: Array<number> } = {};
+    // currently, these store the playlist with the newest creation date
+    const seenNameAndArtistPlaylist: { [key: string]: SpotifyPlaylistType } = {};
+    const seenIdPlaylist: { [key: string]: SpotifyPlaylistType } = {};
+    const result = tracks.reduce((duplicates, playlistTrack, index) => {
+      let track = playlistTrack.track;
+      if (track === null) return duplicates;
+      if (track.id === null) return duplicates;
+      let isDuplicate = false;
+      const seenNameAndArtistKey =
+        `${track.name}:${track.artists[0].name}`.toLowerCase();
+      if (track.id in seenIds) {
+        // if the two tracks have the same Spotify ID, they are duplicates
+        isDuplicate = true;
+      } else {
+        // if they have the same name, main artist, and roughly same duration
+        // we consider tem duplicates too
+        if (seenNameAndArtistKey in seenNameAndArtist) {
+          // we check if _any_ of the previous durations is similar to the one we are checking
+          if (
+            seenNameAndArtist[seenNameAndArtistKey].filter(
+              (duration) => Math.abs(duration - track.duration_ms) < 2000
+            ).length !== 0
+          ) {
+            isDuplicate = true;
+          }
+        }
+      }
+
+      if (isDuplicate) {
+        // assume duplicate younger
+        let firstPlaylist = track.id in seenIds ? seenIdPlaylist[track.id] : seenNameAndArtistPlaylist[seenNameAndArtistKey];
+        let newerPlaylist = playlistTrack.playlist;
+        const insertDate = new Date(playlistTrack.added_at);
+        // check if duplicate younger 
+        if (seenNameAndArtistDate[seenNameAndArtistKey] > insertDate) {
+          firstPlaylist = playlistTrack.playlist;
+          if (track.id in seenIds) {
+            newerPlaylist = seenIdPlaylist[track.id];
+            seenIdPlaylist[track.id] = playlistTrack.playlist;
+          } else {
+            newerPlaylist = seenNameAndArtistPlaylist[seenNameAndArtistKey];
+          }
+          seenNameAndArtistPlaylist[seenNameAndArtistKey] = playlistTrack.playlist;
+          seenNameAndArtistDate[seenNameAndArtistKey] = insertDate;
+          // we have to fix the parent playlist in all existing playlists with the same duplicate. maybe there is a more 
+          // efficient structures for storing duplicates e.g. a tree
+          if (seenNameAndArtistKey in seenNameAndArtist) {
+            duplicates.forEach(dup => {
+              if (`${dup.track.name}:${dup.track.artists[0].name}`.toLowerCase() == seenNameAndArtistKey && dup.firstPlaylist === newerPlaylist) {
+                dup.firstPlaylist = firstPlaylist;
+              }
+            });
+          }
+        }
+        duplicates.push({
+          index: index,
+          track: track,
+          reason: track.id in seenIds ? 'same-id' : 'same-name-artist',
+          firstPlaylist: firstPlaylist,
+          newerPlaylist: newerPlaylist
+        })
+
+      } else {
+        seenIds[track.id] = true;
+        seenNameAndArtist[seenNameAndArtistKey] =
+          seenNameAndArtist[seenNameAndArtistKey] || [];
+        seenNameAndArtist[seenNameAndArtistKey].push(track.duration_ms);
+        seenIdPlaylist[track.id] = playlistTrack.playlist;
+        seenNameAndArtistPlaylist[seenNameAndArtistKey] = playlistTrack.playlist;
+        seenNameAndArtistDate[seenNameAndArtistKey] = new Date(playlistTrack.added_at);
+      }
+      return duplicates;
+    }, []);
+    return result;
+  }
+
+  static async getTracks(
+    api: SpotifyWebApi,
+    playlists: Array<SpotifyPlaylistType>) {
+    let promises = playlists.map(playlistType => PlaylistDeduplicator.getPlaylistTracks(api, playlistType));
+    return Promise.all(promises);
+  }
+}
+
 export class PlaylistDeduplicator extends BaseDeduplicator {
+  static async getPlaylistTracks(
+    api: SpotifyWebApi,
+    playlist: SpotifyPlaylistType
+  ): Promise<Array<SpotifyPlaylistTrackType>> {
+    return new Promise((resolve, reject) => {
+      const playlistTracks = [];
+      promisesForPages(
+        api,
+        api.getGeneric(playlist.tracks.href) // 'https://api.spotify.com/v1/users/11153223185/playlists/0yygtDHfwC7uITHxfrcQsF/tracks'
+      )
+        .then(
+          (
+            pagePromises // todo: I'd love to replace this with
+          ) =>
+            // .then(Promise.all)
+            // à la http://www.html5rocks.com/en/tutorials/es6/promises/#toc-transforming-values
+            Promise.all(pagePromises)
+        )
+        .then((pages) => {
+          pages.forEach((page) => {
+            page.items.forEach((item: SpotifyPlaylistTrackType) => {
+              if (item != null && item.track != null) {
+                item.playlist = playlist;
+                playlistTracks.push(item);
+              }
+            });
+          });
+          resolve(playlistTracks);
+        })
+        .catch(reject);
+    });
+  }
+
   static async getTracks(
     api: SpotifyWebApi,
     playlist: SpotifyPlaylistType