Skip to content

Commit

Permalink
Add crawl limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@totetmatt
totetmatt committed Sep 22, 2023
1 parent 64ec1e8 commit 3f228e8
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 41 deletions.
40 changes: 29 additions & 11 deletions modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.awt.Color;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
Expand Down Expand Up @@ -44,15 +45,17 @@ public class BlueskyGephi {
private final static String NBPREF_QUERY_ISFOLLOWERSACTIVE = "query.isFollowersActive";
private final static String NBPREF_QUERY_ISFOLLOWSACTIVE = "query.isFollowsActive";
private final static String NBPREF_QUERY_ISDEEPSEARCH = "query.isDeepSearch";

private final static String NBPREF_QUERY_ISLIMITCRAWLACTIVE = "query.isLimitCrawlActive";
private final static String NBPREF_QUERY_LIMITCRAWL = "query.limitCrawl";

private final Preferences nbPref = NbPreferences.forModule(BlueskyGephi.class);
// If ATProto get released and decentralized, this will change to adapt to other instances
final private AtClient client = new AtClient("bsky.social");
final private GraphModel graphModel;
private GraphModel graphModel;

public BlueskyGephi() {
initProjectAndWorkspace();
graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel();

}

private void initProjectAndWorkspace() {
Expand Down Expand Up @@ -110,7 +113,21 @@ public void setIsDeepSearch(boolean setIsDeepSearch) {
public boolean getIsDeepSearch() {
return nbPref.getBoolean(NBPREF_QUERY_ISDEEPSEARCH, true);
}

public void setIsLimitCrawlActive(boolean isLimitCrawlActive){
nbPref.putBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, isLimitCrawlActive);
}
public boolean getIsLimitCrawlActive(){
return nbPref.getBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, true);
}


public void setLimitCrawl(int limitCrawl){
nbPref.putInt(NBPREF_QUERY_LIMITCRAWL, limitCrawl);
}
public int getLimitCrawl(){
return nbPref.getInt(NBPREF_QUERY_LIMITCRAWL, 1);
}
private Node createNode(Identity i) {

Node node = graphModel.getGraph().getNode(i.getDid());
Expand Down Expand Up @@ -146,10 +163,10 @@ private void fetchFollowerFollowsFromActor(String actor, List<String> listInit,
private ProgressTicket progressTicket;
Set<String> foaf = new HashSet<>();

private void process(String actor, boolean isDeepSearch) {
private void process(String actor, boolean isDeepSearch, Optional<Integer> limitCrawl) {
try {
if (isFollowsActive) {
List<AppBskyGraphGetFollows> responses = client.appBskyGraphGetFollows(actor);
List<AppBskyGraphGetFollows> responses = client.appBskyGraphGetFollows(actor,limitCrawl);

graphModel.getGraph().writeLock();
for (var response : responses) {
Expand All @@ -169,7 +186,7 @@ private void process(String actor, boolean isDeepSearch) {
}

if (isFollowersActive) {
List<AppBskyGraphGetFollowers> responses = client.appBskyGraphGetFollowers(actor);
List<AppBskyGraphGetFollowers> responses = client.appBskyGraphGetFollowers(actor,limitCrawl);

graphModel.getGraph().writeLock();
for (var response : responses) {
Expand Down Expand Up @@ -214,11 +231,11 @@ public void run() {
this.foaf.addAll(listInit);
}
if(actor!=null){
process(actor, isDeepSearch);
process(actor, isDeepSearch,Optional.empty());
}
if (isDeepSearch) {
if (listInit!=null||isDeepSearch ) {
for (var foafActor : foaf) {
process(foafActor, false);
process(foafActor, false,Optional.of(50));
}
}
Progress.finish(progressTicket);
Expand All @@ -233,23 +250,24 @@ private Stream<String> manageList(String listId) {

}
public void fetchFollowerFollowsFromActors(List<String> actors, boolean isFollowsActive, boolean isFollowersActive, boolean isBlocksActive) {
graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel();
actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor,null, isFollowsActive, isFollowersActive, getIsDeepSearch()));
}


public void fetchFollowerFollowsFromActors(List<String> actors) {
graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel();
actors
.stream()
.sequential()
.filter(x -> !x.contains("app.bsky.graph.list"))
.forEach(actor -> fetchFollowerFollowsFromActor(actor,null, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch()));

List<String> listActor = actors
.stream()
.sequential()
.filter(x -> x.contains("app.bsky.graph.list"))
.flatMap(this::manageList)
.collect(Collectors.toList());

fetchFollowerFollowsFromActor(null,listActor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch());

}
Expand Down
57 changes: 43 additions & 14 deletions modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,28 @@
<Group type="102" alignment="0" attributes="0">
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="1" attributes="0">
<Group type="103" groupAlignment="1" attributes="0">
<Component id="runFetchButton" alignment="0" max="32767" attributes="0"/>
<Component id="jScrollPane1" alignment="0" pref="353" max="32767" attributes="0"/>
<Component id="credentialsPanel" max="32767" attributes="0"/>
</Group>
<EmptySpace min="-2" pref="12" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<Group type="102" attributes="0">
<Component id="fetchLabel" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="isDeepSearch" min="-2" max="-2" attributes="0"/>
<Group type="102" attributes="0">
<Group type="102" alignment="0" attributes="0">
<Component id="isFollowersActivated" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" max="-2" attributes="0"/>
<Component id="isFollowsActivated" min="-2" max="-2" attributes="0"/>
</Group>
<Component id="isDeepSearch" alignment="0" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="32767" attributes="0"/>
<Group type="103" groupAlignment="0" max="-2" attributes="0">
<Component id="limitCrawlSpinner" max="32767" attributes="0"/>
<Component id="limitCrawlCheckbox" max="32767" attributes="0"/>
</Group>
<EmptySpace min="0" pref="0" max="32767" attributes="0"/>
</Group>
<Component id="runFetchButton" alignment="0" max="32767" attributes="0"/>
<Component id="jScrollPane1" alignment="0" pref="353" max="32767" attributes="0"/>
<Component id="credentialsPanel" alignment="1" max="32767" attributes="0"/>
</Group>
<EmptySpace min="-2" pref="12" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
Expand All @@ -60,14 +60,18 @@
<Component id="fetchLabel" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="isFollowersActivated" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="isFollowsActivated" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="limitCrawlCheckbox" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
<Component id="isDeepSearch" min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="isDeepSearch" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="limitCrawlSpinner" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace max="-2" attributes="0"/>
<Component id="jScrollPane1" min="-2" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="runFetchButton" min="-2" max="-2" attributes="0"/>
<EmptySpace pref="53" max="32767" attributes="0"/>
<EmptySpace pref="51" max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
Expand Down Expand Up @@ -245,5 +249,30 @@
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="isDeepSearchActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JCheckBox" name="limitCrawlCheckbox">
<Properties>
<Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="fr/totetmatt/blueskygephi/Bundle.properties" key="BlueskyGephiMainPanel.limitCrawlCheckbox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="limitCrawlCheckboxActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JSpinner" name="limitCrawlSpinner">
<Properties>
<Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="fr/totetmatt/blueskygephi/Bundle.properties" key="BlueskyGephiMainPanel.limitCrawlSpinner.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</Properties>
<AccessibilityProperties>
<Property name="AccessibleContext.accessibleDescription" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor">
<ResourceString bundle="fr/totetmatt/blueskygephi/Bundle.properties" key="BlueskyGephiMainPanel.limitCrawlSpinner.AccessibleContext.accessibleDescription" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, &quot;{key}&quot;)"/>
</Property>
</AccessibilityProperties>
<Events>
<EventHandler event="propertyChange" listener="java.beans.PropertyChangeListener" parameters="java.beans.PropertyChangeEvent" handler="limitCrawlSpinnerPropertyChange"/>
</Events>
</Component>
</SubComponents>
</Form>
61 changes: 49 additions & 12 deletions modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ public BlueskyGephiMainPanel() {
isFollowersActivated.setSelected(blueskyGephi.getIsFollowersActive());
isFollowsActivated.setSelected(blueskyGephi.getIsFollowsActive());
isDeepSearch.setSelected(blueskyGephi.getIsDeepSearch());
limitCrawlCheckbox.setSelected(blueskyGephi.getIsLimitCrawlActive());
limitCrawlSpinner.setValue(blueskyGephi.getLimitCrawl()*100);
limitCrawlSpinner.setEnabled(limitCrawlCheckbox.isSelected());
}

/**
Expand All @@ -73,6 +76,8 @@ private void initComponents() {
isFollowsActivated = new javax.swing.JCheckBox();
runFetchButton = new javax.swing.JButton();
isDeepSearch = new javax.swing.JCheckBox();
limitCrawlCheckbox = new javax.swing.JCheckBox();
limitCrawlSpinner = new javax.swing.JSpinner();

setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.toolTipText")); // NOI18N
setName("Bluesky Gephi"); // NOI18N
Expand Down Expand Up @@ -173,29 +178,44 @@ public void actionPerformed(java.awt.event.ActionEvent evt) {
}
});

org.openide.awt.Mnemonics.setLocalizedText(limitCrawlCheckbox, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlCheckbox.text")); // NOI18N
limitCrawlCheckbox.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
limitCrawlCheckboxActionPerformed(evt);
}
});

limitCrawlSpinner.setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlSpinner.toolTipText")); // NOI18N
limitCrawlSpinner.addPropertyChangeListener(new java.beans.PropertyChangeListener() {
public void propertyChange(java.beans.PropertyChangeEvent evt) {
limitCrawlSpinnerPropertyChange(evt);
}
});

javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this);
this.setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(layout.createSequentialGroup()
.addContainerGap()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup()
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
.addComponent(runFetchButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(jScrollPane1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 353, Short.MAX_VALUE)
.addComponent(credentialsPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGap(12, 12, 12))
.addGroup(layout.createSequentialGroup()
.addComponent(fetchLabel)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(isDeepSearch)
.addGroup(layout.createSequentialGroup()
.addComponent(isFollowersActivated)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(isFollowsActivated)))
.addGap(0, 0, Short.MAX_VALUE))))
.addComponent(isFollowsActivated))
.addComponent(isDeepSearch))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
.addComponent(limitCrawlSpinner)
.addComponent(limitCrawlCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)))
.addComponent(runFetchButton, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 353, Short.MAX_VALUE)
.addComponent(credentialsPanel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
.addGap(12, 12, 12))
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
Expand All @@ -206,15 +226,20 @@ public void actionPerformed(java.awt.event.ActionEvent evt) {
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(fetchLabel)
.addComponent(isFollowersActivated)
.addComponent(isFollowsActivated))
.addComponent(isFollowsActivated)
.addComponent(limitCrawlCheckbox))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(isDeepSearch)
.addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(isDeepSearch)
.addComponent(limitCrawlSpinner, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(runFetchButton)
.addContainerGap(53, Short.MAX_VALUE))
.addContainerGap(51, Short.MAX_VALUE))
);

limitCrawlSpinner.getAccessibleContext().setAccessibleDescription(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlSpinner.AccessibleContext.accessibleDescription")); // NOI18N
}// </editor-fold>//GEN-END:initComponents

private void isFollowersActivatedActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_isFollowersActivatedActionPerformed
Expand Down Expand Up @@ -255,6 +280,16 @@ private void isDeepSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN-
blueskyGephi.setIsDeepSearch(isDeepSearch.isSelected());
}//GEN-LAST:event_isDeepSearchActionPerformed

private void limitCrawlCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitCrawlCheckboxActionPerformed
blueskyGephi.setIsLimitCrawlActive(limitCrawlCheckbox.isSelected());
limitCrawlSpinner.setEnabled(limitCrawlCheckbox.isSelected());

}//GEN-LAST:event_limitCrawlCheckboxActionPerformed

private void limitCrawlSpinnerPropertyChange(java.beans.PropertyChangeEvent evt) {//GEN-FIRST:event_limitCrawlSpinnerPropertyChange
blueskyGephi.setLimitCrawl(Math.max(1,((int)limitCrawlSpinner.getValue())/100));
}//GEN-LAST:event_limitCrawlSpinnerPropertyChange


// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton credentialsConnectButton;
Expand All @@ -269,6 +304,8 @@ private void isDeepSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN-
private javax.swing.JCheckBox isFollowersActivated;
private javax.swing.JCheckBox isFollowsActivated;
private javax.swing.JScrollPane jScrollPane1;
private javax.swing.JCheckBox limitCrawlCheckbox;
private javax.swing.JSpinner limitCrawlSpinner;
private javax.swing.JButton runFetchButton;
// End of variables declaration//GEN-END:variables
}

0 comments on commit 3f228e8

Please sign in to comment.