From 438aaf13c7446f3395dbb7de6cace6081a0303de Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Wed, 6 Sep 2023 18:40:32 +0200 Subject: [PATCH 1/6] V0.1.0 --- modules/BlueskyGephi/README.md | 24 ++ modules/BlueskyGephi/pom.xml | 103 +++++++ .../totetmatt/blueskygephi/BlueskyGephi.java | 215 ++++++++++++++ .../blueskygephi/BlueskyGephiMainPanel.form | 249 ++++++++++++++++ .../blueskygephi/BlueskyGephiMainPanel.java | 274 ++++++++++++++++++ .../blueskygephi/atproto/AtClient.java | 131 +++++++++ .../blueskygephi/atproto/AtContext.java | 44 +++ .../response/AppBskyActorGetProfile.java | 28 ++ .../response/AppBskyGraphGetFollowers.java | 40 +++ .../response/AppBskyGraphGetFollows.java | 40 +++ .../ComAtprotoServerCreateSession.java | 65 +++++ .../atproto/response/common/Identity.java | 66 +++++ .../BlueskyGephiDefaultGraphManipulator.java | 95 ++++++ ...kyGephiDefaultGraphManipulatorBuilder.java | 19 ++ ...BlueskyGephiFollowersGraphManipulator.java | 93 ++++++ .../BlueskyGephiFollowsGraphManipulator.java | 93 ++++++ .../BlueskyGephiMainGraphManipulator.java | 88 ++++++ ...ueskyGephiMainGraphManipulatorBuilder.java | 19 ++ modules/BlueskyGephi/src/main/nbm/manifest.mf | 5 + .../totetmatt/blueskygephi/Bundle.properties | 13 + nbactions.xml | 23 ++ pom.xml | 1 + 22 files changed, 1728 insertions(+) create mode 100644 modules/BlueskyGephi/README.md create mode 100644 modules/BlueskyGephi/pom.xml create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtContext.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyActorGetProfile.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollowers.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollows.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/ComAtprotoServerCreateSession.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Identity.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulator.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulatorBuilder.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowersGraphManipulator.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowsGraphManipulator.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulator.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulatorBuilder.java create mode 100644 modules/BlueskyGephi/src/main/nbm/manifest.mf create mode 100644 modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties diff --git a/modules/BlueskyGephi/README.md b/modules/BlueskyGephi/README.md new file mode 100644 index 0000000000..53998a34c0 --- /dev/null +++ b/modules/BlueskyGephi/README.md @@ -0,0 +1,24 @@ +# Bluesky Gephi + +This plugin allow you to visualize and explore the network of users in bluesky via the atprotocol. + +# Quick start +- Get a bluesky account +- Generate a password https://bsky.app/settings/app-passwords +- Install the plugin in Gephi +- Open Gephi +- Put handle and password information +- Search for yourself +- Graph of your connection should appears. + +# Docs + +Keep in mind current atproto access point from bluesky is quite permissive and might change in the future. + +## Fetch from user +You can fetch network from user from multiple way : +- Put one or multiple (separated by line return) handles or dids inside the plugin textarea and click on "Go!" +- You can right click on a node and select contextual menu item related to the plugin + - *Bluesky Fetch default data* , will fetch network based on the current configuration on the plugin panel + - *Fetch followers only data*, will fetch only the followers of the node + - *Fetch follows only data*, will fetch only the follows of the node \ No newline at end of file diff --git a/modules/BlueskyGephi/pom.xml b/modules/BlueskyGephi/pom.xml new file mode 100644 index 0000000000..18da757722 --- /dev/null +++ b/modules/BlueskyGephi/pom.xml @@ -0,0 +1,103 @@ + + + 4.0.0 + + gephi-plugin-parent + org.gephi + 0.10.0 + + + fr.totetmatt + bluesky-gephi + 0.1.0 + nbm + + Bluesky Gephi + + + + com.fasterxml.jackson.core + jackson-databind + 2.13.4.1 + + + org.netbeans.api + org-openide-awt + + + ${project.parent.groupId} + visualization-api + + + ${project.parent.groupId} + datalab-api + + + org.netbeans.api + org-openide-windows + + + org.netbeans.api + org-netbeans-modules-settings + + + org.netbeans.api + org-openide-util-lookup + + + org.netbeans.api + org-openide-util + + + org.gephi + graph-api + + + org.gephi + project-api + + + org.gephi + desktop-project + + + + org.gephi + utils-longtask + + + + + + + org.apache.netbeans.utilities + nbm-maven-plugin + + skip + Apache 2.0 + totetmatt + matthieu.totet@gmail.com + https://totetmatt.fr + https://github.com/totetmatt/gephi-plugins.git + + + + + + + + + + + + oss-sonatype + oss-sonatype + https://oss.sonatype.org/content/repositories/snapshots/ + + true + + + + + + diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java new file mode 100644 index 0000000000..6166c56fb2 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -0,0 +1,215 @@ +package fr.totetmatt.blueskygephi; + +import fr.totetmatt.blueskygephi.atproto.AtClient; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollowers; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollows; +import fr.totetmatt.blueskygephi.atproto.response.common.Identity; +import java.awt.Color; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Logger; +import java.util.prefs.Preferences; +import org.gephi.graph.api.Edge; +import org.gephi.graph.api.GraphController; +import org.gephi.graph.api.GraphModel; +import org.gephi.graph.api.Node; +import org.gephi.project.api.Project; +import org.gephi.project.api.ProjectController; +import org.gephi.utils.progress.Progress; +import org.gephi.utils.progress.ProgressTicket; +import org.gephi.utils.progress.ProgressTicketProvider; +import org.openide.util.Lookup; +import org.openide.util.NbPreferences; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author totetmatt + */ +@ServiceProvider(service = BlueskyGephi.class) +public class BlueskyGephi { + + protected static final Logger logger = Logger.getLogger(BlueskyGephi.class.getName()); + private final static String NBPREF_BSKY_HANDLE = "bsky.handle"; + private final static String NBPREF_BSKY_PASSWORD = "bsky.password"; + private final static String NBPREF_QUERY = "query"; + private final static String NBPREF_QUERY_ISFOLLOWERSACTIVE = "query.isFollowersActive"; + private final static String NBPREF_QUERY_ISFOLLOWSACTIVE = "query.isFollowsActive"; + private final static String NBPREF_QUERY_ISDEEPSEARCH = "query.isDeepSearch"; + + private final Preferences nbPref = NbPreferences.forModule(BlueskyGephi.class); + // If ATProto get released and decentralized, this will change to adapt to other instances + final private AtClient client = new AtClient("bsky.social"); + final private GraphModel graphModel; + + public BlueskyGephi() { + initProjectAndWorkspace(); + graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); + } + + private void initProjectAndWorkspace() { + ProjectController projectController = Lookup.getDefault().lookup(ProjectController.class); + Project currentProject = projectController.getCurrentProject(); + if (currentProject == null) { + projectController.newProject(); + } + } + + public boolean connect(String handle, String password) { + nbPref.put(NBPREF_BSKY_HANDLE, handle); + nbPref.put(NBPREF_BSKY_PASSWORD, password); + + return client.comAtprotoServerCreateSession(handle, password); + + } + + public String getHandle() { + return nbPref.get(NBPREF_BSKY_HANDLE, ""); + } + + public String getPassword() { + return nbPref.get(NBPREF_BSKY_PASSWORD, ""); + } + + public void setQuery(String query) { + nbPref.put(NBPREF_QUERY, query); + } + + public String getQuery() { + return nbPref.get(NBPREF_QUERY, ""); + } + + public void setIsFollowersActive(boolean isFollowersActive) { + nbPref.putBoolean(NBPREF_QUERY_ISFOLLOWERSACTIVE, isFollowersActive); + } + + public boolean getIsFollowersActive() { + return nbPref.getBoolean(NBPREF_QUERY_ISFOLLOWERSACTIVE, true); + } + + public void setIsFollowsActive(boolean isFollowsActive) { + nbPref.putBoolean(NBPREF_QUERY_ISFOLLOWSACTIVE, isFollowsActive); + } + + public boolean getIsFollowsActive() { + return nbPref.getBoolean(NBPREF_QUERY_ISFOLLOWSACTIVE, true); + } + + public void setIsDeepSearch(boolean setIsDeepSearch) { + nbPref.putBoolean(NBPREF_QUERY_ISDEEPSEARCH, setIsDeepSearch); + } + + public boolean getIsDeepSearch() { + return nbPref.getBoolean(NBPREF_QUERY_ISDEEPSEARCH, true); + } + + private Node createNode(Identity i) { + + Node node = graphModel.getGraph().getNode(i.getDid()); + if (node == null) { + node = graphModel.factory().newNode(i.getDid()); + node.setLabel(i.getHandle()); + node.setSize(10); + node.setColor(Color.GRAY); + node.setX((float) ((0.01 + Math.random()) * 1000) - 500); + node.setY((float) ((0.01 + Math.random()) * 1000) - 500); + graphModel.getGraph().addNode(node); + } + + return node; + } + + private Edge createEdge(Node source, Node target) { + + Edge edge = graphModel.getGraph().getEdge(source, target); + if (edge == null) { + edge = graphModel.factory().newEdge(source, target, true); + edge.setWeight(1.0); + edge.setColor(Color.GRAY); + graphModel.getGraph().addEdge(edge); + } + + return edge; + } + + private void fetchFollowerFollowsFromActor(String actor, boolean isFollowsActive, boolean isFollowersActive, boolean isDeepSearch) { + // To avoid locking Gephi UI + Thread t = new Thread() { + private ProgressTicket progressTicket; + Set foaf = new HashSet<>(); + + private void process(String actor, boolean isDeepSearch) { + if (isFollowsActive) { + List responses = client.appBskyGraphGetFollows(actor); + + graphModel.getGraph().writeLock(); + for (var response : responses) { + Identity subject = response.getSubject(); + Node source = createNode(subject); + for (var follow : response.getFollows()) { + if (isDeepSearch) { + foaf.add(follow.getDid()); + } + Node target = createNode(follow); + createEdge(source, target); + } + + } + graphModel.getGraph().writeUnlock(); + } + + if (isFollowersActive) { + List responses = client.appBskyGraphGetFollowers(actor); + + graphModel.getGraph().writeLock(); + for (var response : responses) { + Identity subject = response.getSubject(); + Node target = createNode(subject); + for (var follower : response.getFollowers()) { + if (isDeepSearch) { + foaf.add(follower.getDid()); + } + Node source = createNode(follower); + createEdge(source, target); + } + } + graphModel.getGraph().writeUnlock(); + } + } + + @Override + public void run() { + this.setName("Bluesky Gephi Fetching Data for " + actor); + progressTicket = Lookup.getDefault() + .lookup(ProgressTicketProvider.class) + .createTicket(this.getName(), () -> { + interrupt(); + Progress.finish(progressTicket); + return true; + }); + + Progress.start(progressTicket); + Progress.switchToIndeterminate(progressTicket); + + process(actor, isDeepSearch); + if (isDeepSearch) { + for (var foafActor : foaf) { + process(foafActor, false); + } + } + Progress.finish(progressTicket); + } + }; + t.start(); + + } + + public void fetchFollowerFollowsFromActors(List actors, boolean isFollowsActive, boolean isFollowersActive, boolean isBlocksActive) { + actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor, isFollowsActive, isFollowersActive, getIsDeepSearch())); + } + + public void fetchFollowerFollowsFromActors(List actors) { + actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); + } +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form new file mode 100644 index 0000000000..e133766b2d --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form @@ -0,0 +1,249 @@ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java new file mode 100644 index 0000000000..aca5d0ba09 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java @@ -0,0 +1,274 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/GUIForms/JPanel.java to edit this template + */ +package fr.totetmatt.blueskygephi; + +import java.awt.Color; +import java.awt.Desktop; +import java.io.IOException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import org.openide.awt.ActionID; +import org.openide.awt.ActionReference; +import org.openide.util.Exceptions; +import org.openide.util.Lookup; +import org.openide.windows.TopComponent; + +/** + * + * @author totetmatt + */ + +@TopComponent.Description(preferredID = "BlueskyGephiMainPanel", + iconBase = "fr/totetmatt/gephi/twitter/twitterlogo.png" +) + + +@ActionReference(path = "Menu/Window", position = 334) +@TopComponent.OpenActionRegistration(displayName = "Bluesky Gephi", + preferredID = "MainTwitterStreamerWindow") +@ActionID(category = "Window", id = "fr.totetmatt.blueskygephi.BlueskyGephiMainPanel") +@TopComponent.Registration(mode = "layoutmode", openAtStartup = true, position=2) +public class BlueskyGephiMainPanel extends TopComponent { + protected static final Logger consoleLogger = Logger.getLogger(BlueskyGephiMainPanel.class.getName()); + private final BlueskyGephi blueskyGephi; + /** + * Creates new form BlueskyGephiMainPanel + */ + public BlueskyGephiMainPanel() { + initComponents(); + blueskyGephi = Lookup.getDefault().lookup(BlueskyGephi.class); + + credentialsHandleField.setText(blueskyGephi.getHandle()); + credentialsPasswordField.setText(blueskyGephi.getPassword()); + handleSearchTextArea.setText(blueskyGephi.getQuery()); + isFollowersActivated.setSelected(blueskyGephi.getIsFollowersActive()); + isFollowsActivated.setSelected(blueskyGephi.getIsFollowsActive()); + isDeepSearch.setSelected(blueskyGephi.getIsDeepSearch()); + } + + /** + * This method is called from within the constructor to initialize the form. + * WARNING: Do NOT modify this code. The content of this method is always + * regenerated by the Form Editor. + */ + @SuppressWarnings("unchecked") + // //GEN-BEGIN:initComponents + private void initComponents() { + + credentialsPanel = new javax.swing.JPanel(); + credentialsHandleLabel = new javax.swing.JLabel(); + credentialsHandleField = new javax.swing.JTextField(); + credentialsPasswordLabel = new javax.swing.JLabel(); + credentialsPasswordField = new javax.swing.JPasswordField(); + credentialsConnectButton = new javax.swing.JButton(); + jScrollPane1 = new javax.swing.JScrollPane(); + handleSearchTextArea = new javax.swing.JTextArea(); + fetchLabel = new javax.swing.JLabel(); + isFollowersActivated = new javax.swing.JCheckBox(); + isFollowsActivated = new javax.swing.JCheckBox(); + runFetchButton = new javax.swing.JButton(); + isDeepSearch = new javax.swing.JCheckBox(); + + setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.toolTipText")); // NOI18N + setName("Bluesky Gephi"); // NOI18N + + credentialsPanel.setBorder(javax.swing.BorderFactory.createTitledBorder(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsPanel.border.title"))); // NOI18N + credentialsPanel.setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.Credentials.toolTipText")); // NOI18N + credentialsPanel.setName("Credentials"); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(credentialsHandleLabel, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsHandleLabel.text")); // NOI18N + + credentialsHandleField.setText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsHandleField.text")); // NOI18N + + credentialsPasswordLabel.setForeground(new java.awt.Color(0, 51, 255)); + org.openide.awt.Mnemonics.setLocalizedText(credentialsPasswordLabel, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsPasswordLabel.text")); // NOI18N + credentialsPasswordLabel.setCursor(new java.awt.Cursor(java.awt.Cursor.HAND_CURSOR)); + credentialsPasswordLabel.addMouseListener(new java.awt.event.MouseAdapter() { + public void mouseClicked(java.awt.event.MouseEvent evt) { + credentialsPasswordLabelMouseClicked(evt); + } + }); + + credentialsPasswordField.setText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsPasswordField.text")); // NOI18N + + org.openide.awt.Mnemonics.setLocalizedText(credentialsConnectButton, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.credentialsConnectButton.text")); // NOI18N + credentialsConnectButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + credentialsConnectButtonActionPerformed(evt); + } + }); + + javax.swing.GroupLayout credentialsPanelLayout = new javax.swing.GroupLayout(credentialsPanel); + credentialsPanel.setLayout(credentialsPanelLayout); + credentialsPanelLayout.setHorizontalGroup( + credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(credentialsPanelLayout.createSequentialGroup() + .addContainerGap() + .addGroup(credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(credentialsConnectButton, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(credentialsPanelLayout.createSequentialGroup() + .addGroup(credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(credentialsPasswordLabel) + .addComponent(credentialsHandleLabel)) + .addGap(18, 18, 18) + .addGroup(credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(credentialsHandleField) + .addComponent(credentialsPasswordField)))) + .addContainerGap()) + ); + credentialsPanelLayout.setVerticalGroup( + credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(credentialsPanelLayout.createSequentialGroup() + .addContainerGap() + .addGroup(credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(credentialsHandleLabel) + .addComponent(credentialsHandleField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addGroup(credentialsPanelLayout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(credentialsPasswordLabel) + .addComponent(credentialsPasswordField, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(credentialsConnectButton) + .addContainerGap(15, Short.MAX_VALUE)) + ); + + handleSearchTextArea.setColumns(20); + handleSearchTextArea.setRows(5); + jScrollPane1.setViewportView(handleSearchTextArea); + + org.openide.awt.Mnemonics.setLocalizedText(fetchLabel, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.fetchLabel.text")); // NOI18N + + isFollowersActivated.setSelected(true); + org.openide.awt.Mnemonics.setLocalizedText(isFollowersActivated, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.isFollowersActivated.text")); // NOI18N + isFollowersActivated.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + isFollowersActivatedActionPerformed(evt); + } + }); + + isFollowsActivated.setSelected(true); + org.openide.awt.Mnemonics.setLocalizedText(isFollowsActivated, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.isFollowsActivated.text")); // NOI18N + isFollowsActivated.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + isFollowsActivatedActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(runFetchButton, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.runFetchButton.text")); // NOI18N + runFetchButton.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + runFetchButtonActionPerformed(evt); + } + }); + + org.openide.awt.Mnemonics.setLocalizedText(isDeepSearch, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.isDeepSearch.text")); // NOI18N + isDeepSearch.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + isDeepSearchActionPerformed(evt); + } + }); + + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); + this.setLayout(layout); + layout.setHorizontalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) + .addComponent(runFetchButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(jScrollPane1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 353, Short.MAX_VALUE) + .addComponent(credentialsPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGap(12, 12, 12)) + .addGroup(layout.createSequentialGroup() + .addComponent(fetchLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(isDeepSearch) + .addGroup(layout.createSequentialGroup() + .addComponent(isFollowersActivated) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addComponent(isFollowsActivated))) + .addGap(0, 0, Short.MAX_VALUE)))) + ); + layout.setVerticalGroup( + layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addGroup(layout.createSequentialGroup() + .addContainerGap() + .addComponent(credentialsPanel, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(fetchLabel) + .addComponent(isFollowersActivated) + .addComponent(isFollowsActivated)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(isDeepSearch) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(runFetchButton) + .addContainerGap(53, Short.MAX_VALUE)) + ); + }// //GEN-END:initComponents + + private void isFollowersActivatedActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_isFollowersActivatedActionPerformed + blueskyGephi.setIsFollowersActive(isFollowersActivated.isSelected()); + }//GEN-LAST:event_isFollowersActivatedActionPerformed + + private void credentialsConnectButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_credentialsConnectButtonActionPerformed + if(blueskyGephi.connect(credentialsHandleField.getText(), String.valueOf(credentialsPasswordField.getPassword()))){ + credentialsConnectButton.setBackground(Color.GREEN); + } else { + credentialsConnectButton.setBackground(Color.RED); + } + }//GEN-LAST:event_credentialsConnectButtonActionPerformed + + private void runFetchButtonActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_runFetchButtonActionPerformed + blueskyGephi.setQuery(handleSearchTextArea.getText()); + List actors = Arrays.asList(handleSearchTextArea.getText().split("\\n")) + .stream() + .map(x -> x.trim()) + .collect(Collectors.toList()); + blueskyGephi.fetchFollowerFollowsFromActors(actors); + + }//GEN-LAST:event_runFetchButtonActionPerformed + + private void isFollowsActivatedActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_isFollowsActivatedActionPerformed + blueskyGephi.setIsFollowsActive(isFollowsActivated.isSelected()); + }//GEN-LAST:event_isFollowsActivatedActionPerformed + + private void credentialsPasswordLabelMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_credentialsPasswordLabelMouseClicked + try { + Desktop.getDesktop().browse(URI.create("https://bsky.app/settings/app-passwords")); // TODO add your handling code here: + } catch (IOException ex) { + Exceptions.printStackTrace(ex); + } + }//GEN-LAST:event_credentialsPasswordLabelMouseClicked + + private void isDeepSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_isDeepSearchActionPerformed + blueskyGephi.setIsDeepSearch(isDeepSearch.isSelected()); + }//GEN-LAST:event_isDeepSearchActionPerformed + + + // Variables declaration - do not modify//GEN-BEGIN:variables + private javax.swing.JButton credentialsConnectButton; + private javax.swing.JTextField credentialsHandleField; + private javax.swing.JLabel credentialsHandleLabel; + private javax.swing.JPanel credentialsPanel; + private javax.swing.JPasswordField credentialsPasswordField; + private javax.swing.JLabel credentialsPasswordLabel; + private javax.swing.JLabel fetchLabel; + private javax.swing.JTextArea handleSearchTextArea; + private javax.swing.JCheckBox isDeepSearch; + private javax.swing.JCheckBox isFollowersActivated; + private javax.swing.JCheckBox isFollowsActivated; + private javax.swing.JScrollPane jScrollPane1; + private javax.swing.JButton runFetchButton; + // End of variables declaration//GEN-END:variables +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java new file mode 100644 index 0000000000..79102a7783 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java @@ -0,0 +1,131 @@ +package fr.totetmatt.blueskygephi.atproto; + +/** + * + * @author totetmatt + */ +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyActorGetProfile; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollowers; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollows; +import fr.totetmatt.blueskygephi.atproto.response.ComAtprotoServerCreateSession; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import org.openide.util.Exceptions; + +public class AtClient { + + private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + private final AtContext context; + + private final HttpClient client = HttpClient.newHttpClient(); + private ComAtprotoServerCreateSession session = null; + + public AtClient(String host) { + context = new AtContext(host); + } + + public boolean comAtprotoServerCreateSession(String identifier, String password) { + try { + HttpRequest request = HttpRequest.newBuilder(context.getURIForLexicon("com.atproto.server.createSession")) + .POST(HttpRequest.BodyPublishers.ofString("{\"identifier\":\"" + identifier + "\",\"password\":\"" + password + "\"}")) + .header("Content-Type", "application/json") + .build(); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + this.session = objectMapper.readValue(response.body(), ComAtprotoServerCreateSession.class); + return true; + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private HttpRequest getRequest(String xrpcMethod, HashMap params) { + return HttpRequest + .newBuilder(context.getURIForLexicon(xrpcMethod, params)) + .GET() + .header("Authorization", "Bearer " + session.getAccessJwt()) + .build(); + } + + // Yeah, it should be generalized, and async, but it works right now so it's ok. + public List appBskyGraphGetFollowers(String actor) { + List pagedResponse = new ArrayList<>(); + try { + var params = new HashMap(); + params.put("actor", actor); + params.put("limit", "100"); + while (true) { + var request = getRequest("app.bsky.graph.getFollowers", params); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + + AppBskyGraphGetFollowers objectResponse = objectMapper.readValue(response.body(), AppBskyGraphGetFollowers.class); + pagedResponse.add(objectResponse); + if (objectResponse.getCursor() == null) { + break; + } + params.put("cursor", objectResponse.getCursor()); + } + return pagedResponse; + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + public List appBskyGraphGetFollows(String actor) { + List pagedResponse = new ArrayList<>(); + try { + var params = new HashMap(); + params.put("actor", actor); + params.put("limit", "100"); + while (true) { + var request = getRequest("app.bsky.graph.getFollows", params); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + var objectResponse = objectMapper.readValue(response.body(), AppBskyGraphGetFollows.class); + pagedResponse.add(objectResponse); + if (objectResponse.getCursor() == null) { + break; + } + params.put("cursor", objectResponse.getCursor()); + } + return pagedResponse; + } catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + public AppBskyActorGetProfile appBskyActorGetProfile(String actor) { + try { + var params = new HashMap(); + params.put("actor", actor); + var request = getRequest("app.bsky.actor.getProfile", params); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + return objectMapper.readValue(response.body(), AppBskyActorGetProfile.class); + } catch (IOException | InterruptedException ex) { + Exceptions.printStackTrace(ex); + } + return null; + } + + public AppBskyActorGetProfile appBskyActorGetProfiles(String actors) { + try { + var request = HttpRequest + .newBuilder(context.getURIForLexicon("app.bsky.actor.getProfiles", actors)) + .GET() + .header("Authorization", "Bearer " + session.getAccessJwt()) + .build(); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println(response.body()); + return objectMapper.readValue(response.body(), AppBskyActorGetProfile.class); + } catch (IOException | InterruptedException ex) { + Exceptions.printStackTrace(ex); + } + return null; + } +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtContext.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtContext.java new file mode 100644 index 0000000000..42c70824af --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtContext.java @@ -0,0 +1,44 @@ +package fr.totetmatt.blueskygephi.atproto; + +/** + * + * @author totetmatt + */ +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.stream.Collectors; + +public class AtContext { + + private final String host; + + public AtContext(String host) { + this.host = host; + } + + private String formatUrl(String host, String lexicon) { + return "https://" + host + "/xrpc/" + lexicon; + } + + public URI getURIForLexicon(String lexicon) { + + return URI.create(formatUrl(host, lexicon)); + } + + public URI getURIForLexicon(String lexicon, HashMap parameters) { + + String url_parameters = parameters.entrySet().stream().map(x + -> URLEncoder.encode(x.getKey(), StandardCharsets.UTF_8) + "=" + URLEncoder.encode(x.getValue(), StandardCharsets.UTF_8) + ).collect(Collectors.joining("&")); + + return URI.create(formatUrl(host, lexicon) + "?" + url_parameters); + } + + public URI getURIForLexicon(String lexicon, String parameters) { + + return URI.create(formatUrl(host, lexicon) + "?" + parameters); + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyActorGetProfile.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyActorGetProfile.java new file mode 100644 index 0000000000..791b02e1d8 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyActorGetProfile.java @@ -0,0 +1,28 @@ +package fr.totetmatt.blueskygephi.atproto.response; + +/** + * + * @author totetmatt + */ +public class AppBskyActorGetProfile { + + private String did; + private String handle; + + public String getDid() { + return did; + } + + public void setDid(String did) { + this.did = did; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollowers.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollowers.java new file mode 100644 index 0000000000..75e095381e --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollowers.java @@ -0,0 +1,40 @@ +package fr.totetmatt.blueskygephi.atproto.response; + +/** + * + * @author totetmatt + */ +import fr.totetmatt.blueskygephi.atproto.response.common.Identity; +import java.util.List; + +public class AppBskyGraphGetFollowers { + + private Identity subject; + + public Identity getSubject() { + return subject; + } + + public void setSubject(Identity subject) { + this.subject = subject; + } + + public List getFollowers() { + return followers; + } + + public void setFollowers(List followers) { + this.followers = followers; + } + + public String getCursor() { + return cursor; + } + + public void setCursor(String cursor) { + this.cursor = cursor; + } + + private List followers; + private String cursor; +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollows.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollows.java new file mode 100644 index 0000000000..950fba0178 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetFollows.java @@ -0,0 +1,40 @@ +package fr.totetmatt.blueskygephi.atproto.response; + +/** + * + * @author totetmatt + */ +import fr.totetmatt.blueskygephi.atproto.response.common.Identity; +import java.util.List; + +public class AppBskyGraphGetFollows { + + private Identity subject; + + public Identity getSubject() { + return subject; + } + + public void setSubject(Identity subject) { + this.subject = subject; + } + + public List getFollows() { + return follows; + } + + public void setFollows(List follows) { + this.follows = follows; + } + + public String getCursor() { + return cursor; + } + + public void setCursor(String cursor) { + this.cursor = cursor; + } + + private List follows; + private String cursor; +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/ComAtprotoServerCreateSession.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/ComAtprotoServerCreateSession.java new file mode 100644 index 0000000000..181032a566 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/ComAtprotoServerCreateSession.java @@ -0,0 +1,65 @@ +package fr.totetmatt.blueskygephi.atproto.response; + +/** + * + * @author totetmatt + */ +public class ComAtprotoServerCreateSession { + + private String did; + private String handle; + private String email; + private String accessJwt; + private String refreshJwt; + + public String getDid() { + return did; + } + + public void setDid(String did) { + this.did = did; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public String getEmail() { + return email; + } + + public void setEmail(String email) { + this.email = email; + } + + public String getAccessJwt() { + return accessJwt; + } + + public void setAccessJwt(String accessJwt) { + this.accessJwt = accessJwt; + } + + public String getRefreshJwt() { + return refreshJwt; + } + + public void setRefreshJwt(String refreshJwt) { + this.refreshJwt = refreshJwt; + } + + @Override + public String toString() { + return "GetSessionResponse{" + + "did='" + did + '\'' + + ", handle='" + handle + '\'' + + ", email='" + email + '\'' + + ", accessJwt='" + accessJwt + '\'' + + ", refreshJwt='" + refreshJwt + '\'' + + '}'; + } +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Identity.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Identity.java new file mode 100644 index 0000000000..dfa3237b97 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Identity.java @@ -0,0 +1,66 @@ +package fr.totetmatt.blueskygephi.atproto.response.common; + +/** + * + * @author totetmatt + */ +public class Identity { + + private String did; + private String handle; + private String description; + + private String avatar; + private String indexedAt; + + public String getDid() { + return did; + } + + public void setDid(String did) { + this.did = did; + } + + public String getHandle() { + return handle; + } + + public void setHandle(String handle) { + this.handle = handle; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getAvatar() { + return avatar; + } + + public void setAvatar(String avatar) { + this.avatar = avatar; + } + + public String getIndexedAt() { + return indexedAt; + } + + public void setIndexedAt(String indexedAt) { + this.indexedAt = indexedAt; + } + + @Override + public String toString() { + return "Identity{" + + "did='" + did + '\'' + + ", handle='" + handle + '\'' + + ", description='" + description + '\'' + + ", avatar='" + avatar + '\'' + + ", indexedAt='" + indexedAt + '\'' + + '}'; + } +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulator.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulator.java new file mode 100644 index 0000000000..50c8103432 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulator.java @@ -0,0 +1,95 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import fr.totetmatt.blueskygephi.BlueskyGephi; +import java.awt.event.KeyEvent; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.swing.Icon; +import org.gephi.datalab.spi.ContextMenuItemManipulator; +import org.gephi.datalab.spi.ManipulatorUI; +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.graph.api.Graph; +import org.gephi.graph.api.Node; +import org.gephi.visualization.spi.GraphContextMenuItem; +import org.openide.util.Lookup; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author totetmatt + */ +@ServiceProvider(service = GraphContextMenuItem.class) +public class BlueskyGephiDefaultGraphManipulator implements NodesManipulator, GraphContextMenuItem { + + private Node[] nodes; + + @Override + public void setup(Node[] nodes, Node node) { + this.nodes = nodes; + } + + @Override + public ContextMenuItemManipulator[] getSubItems() { + return null; + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public Integer getMnemonicKey() { + return KeyEvent.VK_W; + } + + @Override + public void execute() { + List actors = Stream.of(nodes).map(x -> (String) x.getId()).collect(Collectors.toList()); + Lookup.getDefault() + .lookup(BlueskyGephi.class) + .fetchFollowerFollowsFromActors(actors); + } + + @Override + public String getName() { + return "Bluesky Fetch default data"; + } + + @Override + public String getDescription() { + return "Fetch configured network from the selected nodes"; + } + + @Override + public boolean canExecute() { + return true; + } + + @Override + public ManipulatorUI getUI() { + return null; + } + + @Override + public int getType() { + return 200; + } + + @Override + public int getPosition() { + return 200; + } + + @Override + public Icon getIcon() { + return null; + } + + @Override + public void setup(Graph graph, Node[] nodes) { + this.nodes = nodes; + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulatorBuilder.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulatorBuilder.java new file mode 100644 index 0000000000..6ac931f3bc --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiDefaultGraphManipulatorBuilder.java @@ -0,0 +1,19 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.datalab.spi.nodes.NodesManipulatorBuilder; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author totetmatt + */ +@ServiceProvider(service = NodesManipulatorBuilder.class) +public class BlueskyGephiDefaultGraphManipulatorBuilder implements NodesManipulatorBuilder { + + @Override + public NodesManipulator getNodesManipulator() { + return new BlueskyGephiDefaultGraphManipulator(); + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowersGraphManipulator.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowersGraphManipulator.java new file mode 100644 index 0000000000..f4626484b8 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowersGraphManipulator.java @@ -0,0 +1,93 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import fr.totetmatt.blueskygephi.BlueskyGephi; +import java.awt.event.KeyEvent; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.swing.Icon; +import org.gephi.datalab.spi.ContextMenuItemManipulator; +import org.gephi.datalab.spi.ManipulatorUI; +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.graph.api.Graph; +import org.gephi.graph.api.Node; +import org.gephi.visualization.spi.GraphContextMenuItem; +import org.openide.util.Lookup; + +/** + * + * @author totetmatt + */ +public class BlueskyGephiFollowersGraphManipulator implements NodesManipulator, GraphContextMenuItem { + + private Node[] nodes; + + @Override + public void setup(Node[] nodes, Node node) { + this.nodes = nodes; + } + + @Override + public ContextMenuItemManipulator[] getSubItems() { + return null; + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public Integer getMnemonicKey() { + return KeyEvent.VK_C; + } + + @Override + public void execute() { + List actors = Stream.of(nodes).map(x -> (String) x.getId()).collect(Collectors.toList()); + Lookup.getDefault() + .lookup(BlueskyGephi.class) + .fetchFollowerFollowsFromActors(actors, false, true, false); + } + + @Override + public String getName() { + return "Fetch followers only data"; + } + + @Override + public String getDescription() { + return "Fetch only followers network from the selected nodes"; + } + + @Override + public boolean canExecute() { + return true; + } + + @Override + public ManipulatorUI getUI() { + return null; + } + + @Override + public int getType() { + return 200; + } + + @Override + public int getPosition() { + return 200; + } + + @Override + public Icon getIcon() { + return null; + } + + @Override + public void setup(Graph graph, Node[] nodes) { + this.nodes = nodes; + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowsGraphManipulator.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowsGraphManipulator.java new file mode 100644 index 0000000000..5fda7c5a9a --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiFollowsGraphManipulator.java @@ -0,0 +1,93 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import fr.totetmatt.blueskygephi.BlueskyGephi; +import java.awt.event.KeyEvent; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.swing.Icon; +import org.gephi.datalab.spi.ContextMenuItemManipulator; +import org.gephi.datalab.spi.ManipulatorUI; +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.graph.api.Graph; +import org.gephi.graph.api.Node; +import org.gephi.visualization.spi.GraphContextMenuItem; +import org.openide.util.Lookup; + +/** + * + * @author totetmatt + */ +public class BlueskyGephiFollowsGraphManipulator implements NodesManipulator, GraphContextMenuItem { + + private Node[] nodes; + + @Override + public void setup(Node[] nodes, Node node) { + this.nodes = nodes; + } + + @Override + public ContextMenuItemManipulator[] getSubItems() { + return null; + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public Integer getMnemonicKey() { + return KeyEvent.VK_X; + } + + @Override + public void execute() { + List actors = Stream.of(nodes).map(x -> (String) x.getId()).collect(Collectors.toList()); + Lookup.getDefault() + .lookup(BlueskyGephi.class) + .fetchFollowerFollowsFromActors(actors, true, false, false); + } + + @Override + public String getName() { + return "Fetch follows only data"; + } + + @Override + public String getDescription() { + return "Fetch only follows network from the selected nodes"; + } + + @Override + public boolean canExecute() { + return true; + } + + @Override + public ManipulatorUI getUI() { + return null; + } + + @Override + public int getType() { + return 200; + } + + @Override + public int getPosition() { + return 200; + } + + @Override + public Icon getIcon() { + return null; + } + + @Override + public void setup(Graph graph, Node[] nodes) { + this.nodes = nodes; + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulator.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulator.java new file mode 100644 index 0000000000..ca50ecb823 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulator.java @@ -0,0 +1,88 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import java.awt.event.KeyEvent; +import javax.swing.Icon; +import org.gephi.datalab.spi.ContextMenuItemManipulator; +import org.gephi.datalab.spi.ManipulatorUI; +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.graph.api.Graph; +import org.gephi.graph.api.Node; +import org.gephi.visualization.spi.GraphContextMenuItem; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author totetmatt + */ +@ServiceProvider(service = GraphContextMenuItem.class) +public class BlueskyGephiMainGraphManipulator implements NodesManipulator, GraphContextMenuItem { + + @Override + public void setup(Node[] nodes, Node node) { + + } + + @Override + public ContextMenuItemManipulator[] getSubItems() { + return new ContextMenuItemManipulator[]{ + new BlueskyGephiFollowsGraphManipulator(), + new BlueskyGephiFollowersGraphManipulator() + }; + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public Integer getMnemonicKey() { + return KeyEvent.VK_W; + } + + @Override + public void execute() { + + } + + @Override + public String getName() { + return "Bluesky"; + } + + @Override + public String getDescription() { + return "Fetch network from the selected nodes"; + } + + @Override + public boolean canExecute() { + return true; + } + + @Override + public ManipulatorUI getUI() { + return null; + } + + @Override + public int getType() { + return 200; + } + + @Override + public int getPosition() { + return 200; + } + + @Override + public Icon getIcon() { + return null; + } + + @Override + public void setup(Graph graph, Node[] nodes) { + + } + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulatorBuilder.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulatorBuilder.java new file mode 100644 index 0000000000..04fd713f1f --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/graphmanipulator/BlueskyGephiMainGraphManipulatorBuilder.java @@ -0,0 +1,19 @@ +package fr.totetmatt.blueskygephi.graphmanipulator; + +import org.gephi.datalab.spi.nodes.NodesManipulator; +import org.gephi.datalab.spi.nodes.NodesManipulatorBuilder; +import org.openide.util.lookup.ServiceProvider; + +/** + * + * @author totetmatt + */ +@ServiceProvider(service = NodesManipulatorBuilder.class) +public class BlueskyGephiMainGraphManipulatorBuilder implements NodesManipulatorBuilder { + + @Override + public NodesManipulator getNodesManipulator() { + return new BlueskyGephiMainGraphManipulator(); + } + +} diff --git a/modules/BlueskyGephi/src/main/nbm/manifest.mf b/modules/BlueskyGephi/src/main/nbm/manifest.mf new file mode 100644 index 0000000000..873ac14d0a --- /dev/null +++ b/modules/BlueskyGephi/src/main/nbm/manifest.mf @@ -0,0 +1,5 @@ +Manifest-Version: 1.0 +OpenIDE-Module-Name: Bluesky Gephi +OpenIDE-Module-Short-Description: Import network from Bluesky / AtProtocol +OpenIDE-Module-Long-Description: Import network from Bluesky / AtProtocol +OpenIDE-Module-Display-Category: Import diff --git a/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties b/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties new file mode 100644 index 0000000000..af2085875d --- /dev/null +++ b/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties @@ -0,0 +1,13 @@ +BlueskyGephiMainPanel.toolTipText=Bluesky Gephi +BlueskyGephiMainPanel.Credentials.toolTipText=Credentials +BlueskyGephiMainPanel.credentialsPanel.border.title=Credentials +BlueskyGephiMainPanel.credentialsHandleLabel.text=Handle +BlueskyGephiMainPanel.credentialsHandleField.text= +BlueskyGephiMainPanel.credentialsConnectButton.text=Connect +BlueskyGephiMainPanel.runFetchButton.text=Go ! +BlueskyGephiMainPanel.isFollowsActivated.text=Follows +BlueskyGephiMainPanel.isFollowersActivated.text=Followers +BlueskyGephiMainPanel.fetchLabel.text=Fetch +BlueskyGephiMainPanel.credentialsPasswordField.text= +BlueskyGephiMainPanel.credentialsPasswordLabel.text=Password +BlueskyGephiMainPanel.isDeepSearch.text=Fetch also n+1 diff --git a/nbactions.xml b/nbactions.xml index b7b2cafb76..b4d01bd070 100644 --- a/nbactions.xml +++ b/nbactions.xml @@ -18,4 +18,27 @@ -J-Xdebug -J-Xrunjdwp:transport=dt_socket,suspend=n,server=n,address=${jpda.address} + + CUSTOM-clean package + clean package + + clean + package + + + + CUSTOM-org.gephi:gephi-maven-plugin:run + org.gephi:gephi-maven-plugin:run + + org.gephi:gephi-maven-plugin:run + + + + CUSTOM-package org.gephi:gephi-maven-plugin:run + package org.gephi:gephi-maven-plugin:run + + package + org.gephi:gephi-maven-plugin:run + + diff --git a/pom.xml b/pom.xml index afb8f2bf5d..e6c49f3d22 100644 --- a/pom.xml +++ b/pom.xml @@ -12,6 +12,7 @@ + modules/BlueskyGephi From 5dd325747fbee3039baa7bc33a5af87fd5795897 Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Sat, 9 Sep 2023 11:29:46 +0200 Subject: [PATCH 2/6] Add: Mark visited nodes --- modules/BlueskyGephi/README.md | 11 ++++++--- .../totetmatt/blueskygephi/BlueskyGephi.java | 2 ++ nbactions.xml | 23 ------------------- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/modules/BlueskyGephi/README.md b/modules/BlueskyGephi/README.md index 53998a34c0..bbcb1a4100 100644 --- a/modules/BlueskyGephi/README.md +++ b/modules/BlueskyGephi/README.md @@ -19,6 +19,11 @@ Keep in mind current atproto access point from bluesky is quite permissive and m You can fetch network from user from multiple way : - Put one or multiple (separated by line return) handles or dids inside the plugin textarea and click on "Go!" - You can right click on a node and select contextual menu item related to the plugin - - *Bluesky Fetch default data* , will fetch network based on the current configuration on the plugin panel - - *Fetch followers only data*, will fetch only the followers of the node - - *Fetch follows only data*, will fetch only the follows of the node \ No newline at end of file + - **Bluesky Fetch default data** , will fetch network based on the current configuration on the plugin panel + - **Fetch followers only data**, will fetch only the followers of the node + - **Fetch follows only data**, will fetch only the follows of the node + +## Deep Search +By activating **Fetch also n+1**, the plugin will fetch the selected handles network **and also** the network of the handles found. + +/!\ Keep in mind that this can be very long as some users has a long list of followers or follows. /!\ \ No newline at end of file diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java index 6166c56fb2..357e49bdbb 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -147,6 +147,7 @@ private void process(String actor, boolean isDeepSearch) { for (var response : responses) { Identity subject = response.getSubject(); Node source = createNode(subject); + source.setColor(Color.GREEN); for (var follow : response.getFollows()) { if (isDeepSearch) { foaf.add(follow.getDid()); @@ -166,6 +167,7 @@ private void process(String actor, boolean isDeepSearch) { for (var response : responses) { Identity subject = response.getSubject(); Node target = createNode(subject); + target.setColor(Color.GREEN); for (var follower : response.getFollowers()) { if (isDeepSearch) { foaf.add(follower.getDid()); diff --git a/nbactions.xml b/nbactions.xml index b4d01bd070..b7b2cafb76 100644 --- a/nbactions.xml +++ b/nbactions.xml @@ -18,27 +18,4 @@ -J-Xdebug -J-Xrunjdwp:transport=dt_socket,suspend=n,server=n,address=${jpda.address} - - CUSTOM-clean package - clean package - - clean - package - - - - CUSTOM-org.gephi:gephi-maven-plugin:run - org.gephi:gephi-maven-plugin:run - - org.gephi:gephi-maven-plugin:run - - - - CUSTOM-package org.gephi:gephi-maven-plugin:run - package org.gephi:gephi-maven-plugin:run - - package - org.gephi:gephi-maven-plugin:run - - From 64ec1e8c8769f130f8681e194b086ae2520a586e Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Thu, 21 Sep 2023 22:54:59 +0200 Subject: [PATCH 3/6] Feature: Add List crawling --- .../totetmatt/blueskygephi/BlueskyGephi.java | 115 ++++++++++++------ .../blueskygephi/atproto/AtClient.java | 29 ++++- .../atproto/response/AppBskyGraphGetList.java | 38 ++++++ .../atproto/response/common/Subject.java | 22 ++++ 4 files changed, 164 insertions(+), 40 deletions(-) create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetList.java create mode 100644 modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Subject.java diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java index 357e49bdbb..a91f676563 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -3,13 +3,19 @@ import fr.totetmatt.blueskygephi.atproto.AtClient; import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollowers; import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollows; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetList; import fr.totetmatt.blueskygephi.atproto.response.common.Identity; import java.awt.Color; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.prefs.Preferences; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.apache.commons.math3.analysis.FunctionUtils.collector; import org.gephi.graph.api.Edge; import org.gephi.graph.api.GraphController; import org.gephi.graph.api.GraphModel; @@ -19,6 +25,7 @@ import org.gephi.utils.progress.Progress; import org.gephi.utils.progress.ProgressTicket; import org.gephi.utils.progress.ProgressTicketProvider; +import org.openide.util.Exceptions; import org.openide.util.Lookup; import org.openide.util.NbPreferences; import org.openide.util.lookup.ServiceProvider; @@ -133,56 +140,65 @@ private Edge createEdge(Node source, Node target) { return edge; } - private void fetchFollowerFollowsFromActor(String actor, boolean isFollowsActive, boolean isFollowersActive, boolean isDeepSearch) { + private void fetchFollowerFollowsFromActor(String actor, List listInit, boolean isFollowsActive, boolean isFollowersActive, boolean isDeepSearch) { // To avoid locking Gephi UI Thread t = new Thread() { private ProgressTicket progressTicket; Set foaf = new HashSet<>(); private void process(String actor, boolean isDeepSearch) { - if (isFollowsActive) { - List responses = client.appBskyGraphGetFollows(actor); - - graphModel.getGraph().writeLock(); - for (var response : responses) { - Identity subject = response.getSubject(); - Node source = createNode(subject); - source.setColor(Color.GREEN); - for (var follow : response.getFollows()) { - if (isDeepSearch) { - foaf.add(follow.getDid()); + try { + if (isFollowsActive) { + List responses = client.appBskyGraphGetFollows(actor); + + graphModel.getGraph().writeLock(); + for (var response : responses) { + Identity subject = response.getSubject(); + Node source = createNode(subject); + source.setColor(Color.GREEN); + for (var follow : response.getFollows()) { + if (isDeepSearch) { + foaf.add(follow.getDid()); + } + Node target = createNode(follow); + createEdge(source, target); } - Node target = createNode(follow); - createEdge(source, target); - } + } + graphModel.getGraph().writeUnlock(); } - graphModel.getGraph().writeUnlock(); - } - if (isFollowersActive) { - List responses = client.appBskyGraphGetFollowers(actor); - - graphModel.getGraph().writeLock(); - for (var response : responses) { - Identity subject = response.getSubject(); - Node target = createNode(subject); - target.setColor(Color.GREEN); - for (var follower : response.getFollowers()) { - if (isDeepSearch) { - foaf.add(follower.getDid()); + if (isFollowersActive) { + List responses = client.appBskyGraphGetFollowers(actor); + + graphModel.getGraph().writeLock(); + for (var response : responses) { + Identity subject = response.getSubject(); + Node target = createNode(subject); + target.setColor(Color.GREEN); + for (var follower : response.getFollowers()) { + if (isDeepSearch) { + foaf.add(follower.getDid()); + } + Node source = createNode(follower); + createEdge(source, target); } - Node source = createNode(follower); - createEdge(source, target); } + graphModel.getGraph().writeUnlock(); } - graphModel.getGraph().writeUnlock(); + } catch(Exception e){ + Exceptions.printStackTrace(e); } } @Override public void run() { - this.setName("Bluesky Gephi Fetching Data for " + actor); + + if(actor!=null){ + this.setName("Bluesky Gephi Fetching Data for " + actor); + } else { + this.setName("Bluesky Gephi Fetching Data for List"); + } progressTicket = Lookup.getDefault() .lookup(ProgressTicketProvider.class) .createTicket(this.getName(), () -> { @@ -190,11 +206,16 @@ public void run() { Progress.finish(progressTicket); return true; }); - Progress.start(progressTicket); Progress.switchToIndeterminate(progressTicket); - - process(actor, isDeepSearch); + + + if(listInit!=null){ + this.foaf.addAll(listInit); + } + if(actor!=null){ + process(actor, isDeepSearch); + } if (isDeepSearch) { for (var foafActor : foaf) { process(foafActor, false); @@ -206,12 +227,30 @@ public void run() { t.start(); } - + private Stream manageList(String listId) { + List list = client.appBskyGraphGetList(listId); + return list.stream().flatMap(x->x.getItems().stream().map(y->y.getSubject().getDid())); + + } public void fetchFollowerFollowsFromActors(List actors, boolean isFollowsActive, boolean isFollowersActive, boolean isBlocksActive) { - actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor, isFollowsActive, isFollowersActive, getIsDeepSearch())); + actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor,null, isFollowsActive, isFollowersActive, getIsDeepSearch())); } + public void fetchFollowerFollowsFromActors(List actors) { - actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); + actors + .stream() + .sequential() + .filter(x -> !x.contains("app.bsky.graph.list")) + .forEach(actor -> fetchFollowerFollowsFromActor(actor,null, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); + + List listActor = actors + .stream() + .sequential() + .filter(x -> x.contains("app.bsky.graph.list")) + .flatMap(this::manageList) + .collect(Collectors.toList()); + fetchFollowerFollowsFromActor(null,listActor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch()); + } } diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java index 79102a7783..00cdf68992 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java @@ -9,6 +9,7 @@ import fr.totetmatt.blueskygephi.atproto.response.AppBskyActorGetProfile; import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollowers; import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetFollows; +import fr.totetmatt.blueskygephi.atproto.response.AppBskyGraphGetList; import fr.totetmatt.blueskygephi.atproto.response.ComAtprotoServerCreateSession; import java.io.IOException; import java.net.http.HttpClient; @@ -17,6 +18,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.openide.util.Exceptions; public class AtClient { @@ -26,7 +29,7 @@ public class AtClient { private final HttpClient client = HttpClient.newHttpClient(); private ComAtprotoServerCreateSession session = null; - +ExecutorService executorService = Executors.newFixedThreadPool(2); public AtClient(String host) { context = new AtContext(host); } @@ -112,6 +115,29 @@ public AppBskyActorGetProfile appBskyActorGetProfile(String actor) { } return null; } + + public List appBskyGraphGetList(String list) { + List lists = new ArrayList<>(); + try { + var params = new HashMap(); + params.put("list", list); + params.put("limit", "100"); + while (true) { + var request = getRequest("app.bsky.graph.getList", params); + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + var objectResponse = objectMapper.readValue(response.body(), AppBskyGraphGetList.class); + lists.add(objectResponse); + if (objectResponse.getCursor() == null) { + break; + } + + params.put("cursor", objectResponse.getCursor()); + } + } catch (IOException | InterruptedException ex) { + Exceptions.printStackTrace(ex); + } + return lists; + } public AppBskyActorGetProfile appBskyActorGetProfiles(String actors) { try { @@ -121,7 +147,6 @@ public AppBskyActorGetProfile appBskyActorGetProfiles(String actors) { .header("Authorization", "Bearer " + session.getAccessJwt()) .build(); var response = client.send(request, HttpResponse.BodyHandlers.ofString()); - System.out.println(response.body()); return objectMapper.readValue(response.body(), AppBskyActorGetProfile.class); } catch (IOException | InterruptedException ex) { Exceptions.printStackTrace(ex); diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetList.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetList.java new file mode 100644 index 0000000000..7702d8291a --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/AppBskyGraphGetList.java @@ -0,0 +1,38 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package fr.totetmatt.blueskygephi.atproto.response; + +import fr.totetmatt.blueskygephi.atproto.response.common.Subject; +import java.util.List; + +/** + * + * @author totetmatt + */ +public class AppBskyGraphGetList { + private List items; + private String cursor; + + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public String getCursor() { + return cursor; + } + + public void setCursor(String cursor) { + this.cursor = cursor; + } + + + + +} diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Subject.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Subject.java new file mode 100644 index 0000000000..e0297632e4 --- /dev/null +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/response/common/Subject.java @@ -0,0 +1,22 @@ +/* + * Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license + * Click nbfs://nbhost/SystemFileSystem/Templates/Classes/Class.java to edit this template + */ +package fr.totetmatt.blueskygephi.atproto.response.common; + +/** + * + * @author totetmatt + */ +public class Subject { + private Identity subject; + + public Identity getSubject() { + return subject; + } + + public void setSubject(Identity subject) { + this.subject = subject; + } + +} From 3f228e84440c77524e83498d5f1338ec13c8c6f5 Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Fri, 22 Sep 2023 14:59:23 +0200 Subject: [PATCH 4/6] Add crawl limit --- .../totetmatt/blueskygephi/BlueskyGephi.java | 40 ++++++++---- .../blueskygephi/BlueskyGephiMainPanel.form | 57 ++++++++++++----- .../blueskygephi/BlueskyGephiMainPanel.java | 61 +++++++++++++++---- .../blueskygephi/atproto/AtClient.java | 14 +++-- .../totetmatt/blueskygephi/Bundle.properties | 3 + 5 files changed, 134 insertions(+), 41 deletions(-) diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java index a91f676563..59b340435b 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -8,6 +8,7 @@ import java.awt.Color; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -44,15 +45,17 @@ public class BlueskyGephi { private final static String NBPREF_QUERY_ISFOLLOWERSACTIVE = "query.isFollowersActive"; private final static String NBPREF_QUERY_ISFOLLOWSACTIVE = "query.isFollowsActive"; private final static String NBPREF_QUERY_ISDEEPSEARCH = "query.isDeepSearch"; - + private final static String NBPREF_QUERY_ISLIMITCRAWLACTIVE = "query.isLimitCrawlActive"; + private final static String NBPREF_QUERY_LIMITCRAWL = "query.limitCrawl"; + private final Preferences nbPref = NbPreferences.forModule(BlueskyGephi.class); // If ATProto get released and decentralized, this will change to adapt to other instances final private AtClient client = new AtClient("bsky.social"); - final private GraphModel graphModel; + private GraphModel graphModel; public BlueskyGephi() { initProjectAndWorkspace(); - graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); + } private void initProjectAndWorkspace() { @@ -110,7 +113,21 @@ public void setIsDeepSearch(boolean setIsDeepSearch) { public boolean getIsDeepSearch() { return nbPref.getBoolean(NBPREF_QUERY_ISDEEPSEARCH, true); } + + public void setIsLimitCrawlActive(boolean isLimitCrawlActive){ + nbPref.putBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, isLimitCrawlActive); + } + public boolean getIsLimitCrawlActive(){ + return nbPref.getBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, true); + } + + public void setLimitCrawl(int limitCrawl){ + nbPref.putInt(NBPREF_QUERY_LIMITCRAWL, limitCrawl); + } + public int getLimitCrawl(){ + return nbPref.getInt(NBPREF_QUERY_LIMITCRAWL, 1); + } private Node createNode(Identity i) { Node node = graphModel.getGraph().getNode(i.getDid()); @@ -146,10 +163,10 @@ private void fetchFollowerFollowsFromActor(String actor, List listInit, private ProgressTicket progressTicket; Set foaf = new HashSet<>(); - private void process(String actor, boolean isDeepSearch) { + private void process(String actor, boolean isDeepSearch, Optional limitCrawl) { try { if (isFollowsActive) { - List responses = client.appBskyGraphGetFollows(actor); + List responses = client.appBskyGraphGetFollows(actor,limitCrawl); graphModel.getGraph().writeLock(); for (var response : responses) { @@ -169,7 +186,7 @@ private void process(String actor, boolean isDeepSearch) { } if (isFollowersActive) { - List responses = client.appBskyGraphGetFollowers(actor); + List responses = client.appBskyGraphGetFollowers(actor,limitCrawl); graphModel.getGraph().writeLock(); for (var response : responses) { @@ -214,11 +231,11 @@ public void run() { this.foaf.addAll(listInit); } if(actor!=null){ - process(actor, isDeepSearch); + process(actor, isDeepSearch,Optional.empty()); } - if (isDeepSearch) { + if (listInit!=null||isDeepSearch ) { for (var foafActor : foaf) { - process(foafActor, false); + process(foafActor, false,Optional.of(50)); } } Progress.finish(progressTicket); @@ -233,23 +250,24 @@ private Stream manageList(String listId) { } public void fetchFollowerFollowsFromActors(List actors, boolean isFollowsActive, boolean isFollowersActive, boolean isBlocksActive) { + graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor,null, isFollowsActive, isFollowersActive, getIsDeepSearch())); } public void fetchFollowerFollowsFromActors(List actors) { + graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); actors .stream() - .sequential() .filter(x -> !x.contains("app.bsky.graph.list")) .forEach(actor -> fetchFollowerFollowsFromActor(actor,null, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); List listActor = actors .stream() - .sequential() .filter(x -> x.contains("app.bsky.graph.list")) .flatMap(this::manageList) .collect(Collectors.toList()); + fetchFollowerFollowsFromActor(null,listActor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch()); } diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form index e133766b2d..42402520a0 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.form @@ -25,28 +25,28 @@ - - - - - - - - - + - - + + + + + + + - + + + + @@ -60,14 +60,18 @@ + - + + + + - + @@ -245,5 +249,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java index aca5d0ba09..0cb5f9bfde 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephiMainPanel.java @@ -49,6 +49,9 @@ public BlueskyGephiMainPanel() { isFollowersActivated.setSelected(blueskyGephi.getIsFollowersActive()); isFollowsActivated.setSelected(blueskyGephi.getIsFollowsActive()); isDeepSearch.setSelected(blueskyGephi.getIsDeepSearch()); + limitCrawlCheckbox.setSelected(blueskyGephi.getIsLimitCrawlActive()); + limitCrawlSpinner.setValue(blueskyGephi.getLimitCrawl()*100); + limitCrawlSpinner.setEnabled(limitCrawlCheckbox.isSelected()); } /** @@ -73,6 +76,8 @@ private void initComponents() { isFollowsActivated = new javax.swing.JCheckBox(); runFetchButton = new javax.swing.JButton(); isDeepSearch = new javax.swing.JCheckBox(); + limitCrawlCheckbox = new javax.swing.JCheckBox(); + limitCrawlSpinner = new javax.swing.JSpinner(); setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.toolTipText")); // NOI18N setName("Bluesky Gephi"); // NOI18N @@ -173,6 +178,20 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { } }); + org.openide.awt.Mnemonics.setLocalizedText(limitCrawlCheckbox, org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlCheckbox.text")); // NOI18N + limitCrawlCheckbox.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + limitCrawlCheckboxActionPerformed(evt); + } + }); + + limitCrawlSpinner.setToolTipText(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlSpinner.toolTipText")); // NOI18N + limitCrawlSpinner.addPropertyChangeListener(new java.beans.PropertyChangeListener() { + public void propertyChange(java.beans.PropertyChangeEvent evt) { + limitCrawlSpinnerPropertyChange(evt); + } + }); + javax.swing.GroupLayout layout = new javax.swing.GroupLayout(this); this.setLayout(layout); layout.setHorizontalGroup( @@ -180,22 +199,23 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { .addGroup(layout.createSequentialGroup() .addContainerGap() .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, layout.createSequentialGroup() - .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) - .addComponent(runFetchButton, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) - .addComponent(jScrollPane1, javax.swing.GroupLayout.Alignment.LEADING, javax.swing.GroupLayout.DEFAULT_SIZE, 353, Short.MAX_VALUE) - .addComponent(credentialsPanel, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) - .addGap(12, 12, 12)) .addGroup(layout.createSequentialGroup() .addComponent(fetchLabel) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) - .addComponent(isDeepSearch) .addGroup(layout.createSequentialGroup() .addComponent(isFollowersActivated) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) - .addComponent(isFollowsActivated))) - .addGap(0, 0, Short.MAX_VALUE)))) + .addComponent(isFollowsActivated)) + .addComponent(isDeepSearch)) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false) + .addComponent(limitCrawlSpinner) + .addComponent(limitCrawlCheckbox, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))) + .addComponent(runFetchButton, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE) + .addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 353, Short.MAX_VALUE) + .addComponent(credentialsPanel, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) + .addGap(12, 12, 12)) ); layout.setVerticalGroup( layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) @@ -206,15 +226,20 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) .addComponent(fetchLabel) .addComponent(isFollowersActivated) - .addComponent(isFollowsActivated)) + .addComponent(isFollowsActivated) + .addComponent(limitCrawlCheckbox)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) - .addComponent(isDeepSearch) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE) + .addComponent(isDeepSearch) + .addComponent(limitCrawlSpinner, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(runFetchButton) - .addContainerGap(53, Short.MAX_VALUE)) + .addContainerGap(51, Short.MAX_VALUE)) ); + + limitCrawlSpinner.getAccessibleContext().setAccessibleDescription(org.openide.util.NbBundle.getMessage(BlueskyGephiMainPanel.class, "BlueskyGephiMainPanel.limitCrawlSpinner.AccessibleContext.accessibleDescription")); // NOI18N }// //GEN-END:initComponents private void isFollowersActivatedActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_isFollowersActivatedActionPerformed @@ -255,6 +280,16 @@ private void isDeepSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN- blueskyGephi.setIsDeepSearch(isDeepSearch.isSelected()); }//GEN-LAST:event_isDeepSearchActionPerformed + private void limitCrawlCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_limitCrawlCheckboxActionPerformed + blueskyGephi.setIsLimitCrawlActive(limitCrawlCheckbox.isSelected()); + limitCrawlSpinner.setEnabled(limitCrawlCheckbox.isSelected()); + + }//GEN-LAST:event_limitCrawlCheckboxActionPerformed + + private void limitCrawlSpinnerPropertyChange(java.beans.PropertyChangeEvent evt) {//GEN-FIRST:event_limitCrawlSpinnerPropertyChange + blueskyGephi.setLimitCrawl(Math.max(1,((int)limitCrawlSpinner.getValue())/100)); + }//GEN-LAST:event_limitCrawlSpinnerPropertyChange + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JButton credentialsConnectButton; @@ -269,6 +304,8 @@ private void isDeepSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN- private javax.swing.JCheckBox isFollowersActivated; private javax.swing.JCheckBox isFollowsActivated; private javax.swing.JScrollPane jScrollPane1; + private javax.swing.JCheckBox limitCrawlCheckbox; + private javax.swing.JSpinner limitCrawlSpinner; private javax.swing.JButton runFetchButton; // End of variables declaration//GEN-END:variables } diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java index 00cdf68992..4281f67438 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/atproto/AtClient.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import org.openide.util.Exceptions; @@ -57,13 +58,14 @@ private HttpRequest getRequest(String xrpcMethod, HashMap params } // Yeah, it should be generalized, and async, but it works right now so it's ok. - public List appBskyGraphGetFollowers(String actor) { + public List appBskyGraphGetFollowers(String actor, Optional limitCrawl) { List pagedResponse = new ArrayList<>(); try { var params = new HashMap(); params.put("actor", actor); params.put("limit", "100"); - while (true) { + int currentCrawlLoop=0; + while (limitCrawl.isEmpty() ||currentCrawlLoop < limitCrawl.get()) { var request = getRequest("app.bsky.graph.getFollowers", params); var response = client.send(request, HttpResponse.BodyHandlers.ofString()); @@ -73,6 +75,7 @@ public List appBskyGraphGetFollowers(String actor) { break; } params.put("cursor", objectResponse.getCursor()); + currentCrawlLoop++; } return pagedResponse; } catch (IOException | InterruptedException e) { @@ -80,13 +83,14 @@ public List appBskyGraphGetFollowers(String actor) { } } - public List appBskyGraphGetFollows(String actor) { + public List appBskyGraphGetFollows(String actor, Optional limitCrawl) { List pagedResponse = new ArrayList<>(); try { var params = new HashMap(); params.put("actor", actor); params.put("limit", "100"); - while (true) { + int currentCrawlLoop=0; + while (limitCrawl.isEmpty() || currentCrawlLoop < limitCrawl.get()) { var request = getRequest("app.bsky.graph.getFollows", params); var response = client.send(request, HttpResponse.BodyHandlers.ofString()); var objectResponse = objectMapper.readValue(response.body(), AppBskyGraphGetFollows.class); @@ -95,6 +99,7 @@ public List appBskyGraphGetFollows(String actor) { break; } params.put("cursor", objectResponse.getCursor()); + currentCrawlLoop++; } return pagedResponse; } catch (IOException | InterruptedException e) { @@ -127,6 +132,7 @@ public List appBskyGraphGetList(String list) { var response = client.send(request, HttpResponse.BodyHandlers.ofString()); var objectResponse = objectMapper.readValue(response.body(), AppBskyGraphGetList.class); lists.add(objectResponse); + System.out.println(response.body()); if (objectResponse.getCursor() == null) { break; } diff --git a/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties b/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties index af2085875d..30123d975d 100644 --- a/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties +++ b/modules/BlueskyGephi/src/main/resources/fr/totetmatt/blueskygephi/Bundle.properties @@ -11,3 +11,6 @@ BlueskyGephiMainPanel.fetchLabel.text=Fetch BlueskyGephiMainPanel.credentialsPasswordField.text= BlueskyGephiMainPanel.credentialsPasswordLabel.text=Password BlueskyGephiMainPanel.isDeepSearch.text=Fetch also n+1 +BlueskyGephiMainPanel.limitCrawlCheckbox.text=Crawl Limit +BlueskyGephiMainPanel.limitCrawlSpinner.AccessibleContext.accessibleDescription= +BlueskyGephiMainPanel.limitCrawlSpinner.toolTipText=Nb Follower / Following to fetch max on a n+1 crawl From 7420b3a9f963b740f6fdccc066bdcd2e7c276445 Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Sat, 23 Sep 2023 09:48:12 +0200 Subject: [PATCH 5/6] Enhancement for filter + readme --- modules/BlueskyGephi/README.md | 12 ++++++- .../totetmatt/blueskygephi/BlueskyGephi.java | 36 ++++++++++++------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/modules/BlueskyGephi/README.md b/modules/BlueskyGephi/README.md index bbcb1a4100..bb4757f7b5 100644 --- a/modules/BlueskyGephi/README.md +++ b/modules/BlueskyGephi/README.md @@ -26,4 +26,14 @@ You can fetch network from user from multiple way : ## Deep Search By activating **Fetch also n+1**, the plugin will fetch the selected handles network **and also** the network of the handles found. -/!\ Keep in mind that this can be very long as some users has a long list of followers or follows. /!\ \ No newline at end of file +/!\ Keep in mind that this can be very long as some users has a long list of followers or follows. /!\ + +## Crawl Limit +To have the list of the followers and follows of a user, the atproto api is build in a way that the application need to loop over multiple +"pages". It means that for hub user, that have a high number of followers and follows (10k, 100k,1M) it might take an important amount of time +to retrive information for this kind of user. + +Therefore, there is a possibility to limit this by only retriving a fraction of the followers and follows in order to speedup the exploration. + +It's ok to do that if analysing theses hub isn't your main goal, as if theses hub are highly connected, they will automatically appears on the relationship +of other users. \ No newline at end of file diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java index 59b340435b..7fed2d4a48 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -10,13 +10,10 @@ import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.logging.Logger; import java.util.prefs.Preferences; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.commons.math3.analysis.FunctionUtils.collector; import org.gephi.graph.api.Edge; import org.gephi.graph.api.GraphController; import org.gephi.graph.api.GraphModel; @@ -126,7 +123,7 @@ public void setLimitCrawl(int limitCrawl){ nbPref.putInt(NBPREF_QUERY_LIMITCRAWL, limitCrawl); } public int getLimitCrawl(){ - return nbPref.getInt(NBPREF_QUERY_LIMITCRAWL, 1); + return nbPref.getInt(NBPREF_QUERY_LIMITCRAWL, 50); } private Node createNode(Identity i) { @@ -164,12 +161,14 @@ private void fetchFollowerFollowsFromActor(String actor, List listInit, Set foaf = new HashSet<>(); private void process(String actor, boolean isDeepSearch, Optional limitCrawl) { + try { if (isFollowsActive) { List responses = client.appBskyGraphGetFollows(actor,limitCrawl); - graphModel.getGraph().writeLock(); + for (var response : responses) { + graphModel.getGraph().writeLock(); Identity subject = response.getSubject(); Node source = createNode(subject); source.setColor(Color.GREEN); @@ -180,16 +179,19 @@ private void process(String actor, boolean isDeepSearch, Optional limit Node target = createNode(follow); createEdge(source, target); } + graphModel.getGraph().writeUnlock(); } - graphModel.getGraph().writeUnlock(); + + } if (isFollowersActive) { List responses = client.appBskyGraphGetFollowers(actor,limitCrawl); - graphModel.getGraph().writeLock(); + for (var response : responses) { + graphModel.getGraph().writeLock(); Identity subject = response.getSubject(); Node target = createNode(subject); target.setColor(Color.GREEN); @@ -200,11 +202,13 @@ private void process(String actor, boolean isDeepSearch, Optional limit Node source = createNode(follower); createEdge(source, target); } + graphModel.getGraph().writeUnlock(); } - graphModel.getGraph().writeUnlock(); + } } catch(Exception e){ Exceptions.printStackTrace(e); + } finally { } } @@ -212,9 +216,9 @@ private void process(String actor, boolean isDeepSearch, Optional limit public void run() { if(actor!=null){ - this.setName("Bluesky Gephi Fetching Data for " + actor); + this.setName("[Bsky] fetching" + actor); } else { - this.setName("Bluesky Gephi Fetching Data for List"); + this.setName("[Bsky] fetching List"); } progressTicket = Lookup.getDefault() .lookup(ProgressTicketProvider.class) @@ -225,7 +229,7 @@ public void run() { }); Progress.start(progressTicket); Progress.switchToIndeterminate(progressTicket); - + if(listInit!=null){ this.foaf.addAll(listInit); @@ -234,8 +238,16 @@ public void run() { process(actor, isDeepSearch,Optional.empty()); } if (listInit!=null||isDeepSearch ) { + Progress.switchToDeterminate(progressTicket, foaf.size()); for (var foafActor : foaf) { - process(foafActor, false,Optional.of(50)); + Progress.setDisplayName(progressTicket, "[Bsky] fetching "+actor+" n+1 > "+foafActor); + if(getIsLimitCrawlActive()){ + process(foafActor, false,Optional.of(getLimitCrawl())); + } else { + process(foafActor, false,Optional.empty()); + } + Progress.progress(progressTicket); + } } Progress.finish(progressTicket); From 49995c58cb9948f24a6b3f56b9acc9068ade221c Mon Sep 17 00:00:00 2001 From: Matthieu Totet Date: Mon, 30 Oct 2023 22:45:11 +0100 Subject: [PATCH 6/6] Add account description on node properties --- .../totetmatt/blueskygephi/BlueskyGephi.java | 105 ++++++++++-------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java index 7fed2d4a48..7efadd9f4a 100644 --- a/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java +++ b/modules/BlueskyGephi/src/main/java/fr/totetmatt/blueskygephi/BlueskyGephi.java @@ -44,7 +44,7 @@ public class BlueskyGephi { private final static String NBPREF_QUERY_ISDEEPSEARCH = "query.isDeepSearch"; private final static String NBPREF_QUERY_ISLIMITCRAWLACTIVE = "query.isLimitCrawlActive"; private final static String NBPREF_QUERY_LIMITCRAWL = "query.limitCrawl"; - + private final Preferences nbPref = NbPreferences.forModule(BlueskyGephi.class); // If ATProto get released and decentralized, this will change to adapt to other instances final private AtClient client = new AtClient("bsky.social"); @@ -52,7 +52,7 @@ public class BlueskyGephi { public BlueskyGephi() { initProjectAndWorkspace(); - + } private void initProjectAndWorkspace() { @@ -110,27 +110,30 @@ public void setIsDeepSearch(boolean setIsDeepSearch) { public boolean getIsDeepSearch() { return nbPref.getBoolean(NBPREF_QUERY_ISDEEPSEARCH, true); } - - public void setIsLimitCrawlActive(boolean isLimitCrawlActive){ - nbPref.putBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, isLimitCrawlActive); + + public void setIsLimitCrawlActive(boolean isLimitCrawlActive) { + nbPref.putBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, isLimitCrawlActive); } - public boolean getIsLimitCrawlActive(){ + + public boolean getIsLimitCrawlActive() { return nbPref.getBoolean(NBPREF_QUERY_ISLIMITCRAWLACTIVE, true); } - - public void setLimitCrawl(int limitCrawl){ - nbPref.putInt(NBPREF_QUERY_LIMITCRAWL, limitCrawl); + public void setLimitCrawl(int limitCrawl) { + nbPref.putInt(NBPREF_QUERY_LIMITCRAWL, limitCrawl); } - public int getLimitCrawl(){ + + public int getLimitCrawl() { return nbPref.getInt(NBPREF_QUERY_LIMITCRAWL, 50); } + private Node createNode(Identity i) { Node node = graphModel.getGraph().getNode(i.getDid()); if (node == null) { node = graphModel.factory().newNode(i.getDid()); node.setLabel(i.getHandle()); + node.setAttribute("Description", i.getDescription()); node.setSize(10); node.setColor(Color.GRAY); node.setX((float) ((0.01 + Math.random()) * 1000) - 500); @@ -161,12 +164,11 @@ private void fetchFollowerFollowsFromActor(String actor, List listInit, Set foaf = new HashSet<>(); private void process(String actor, boolean isDeepSearch, Optional limitCrawl) { - + try { if (isFollowsActive) { - List responses = client.appBskyGraphGetFollows(actor,limitCrawl); + List responses = client.appBskyGraphGetFollows(actor, limitCrawl); - for (var response : responses) { graphModel.getGraph().writeLock(); Identity subject = response.getSubject(); @@ -182,14 +184,12 @@ private void process(String actor, boolean isDeepSearch, Optional limit graphModel.getGraph().writeUnlock(); } - - + } if (isFollowersActive) { - List responses = client.appBskyGraphGetFollowers(actor,limitCrawl); + List responses = client.appBskyGraphGetFollowers(actor, limitCrawl); - for (var response : responses) { graphModel.getGraph().writeLock(); Identity subject = response.getSubject(); @@ -204,21 +204,21 @@ private void process(String actor, boolean isDeepSearch, Optional limit } graphModel.getGraph().writeUnlock(); } - + } - } catch(Exception e){ - Exceptions.printStackTrace(e); + } catch (Exception e) { + Exceptions.printStackTrace(e); } finally { } } @Override public void run() { - - if(actor!=null){ - this.setName("[Bsky] fetching" + actor); + + if (actor != null) { + this.setName("[Bsky] fetching" + actor); } else { - this.setName("[Bsky] fetching List"); + this.setName("[Bsky] fetching List"); } progressTicket = Lookup.getDefault() .lookup(ProgressTicketProvider.class) @@ -229,25 +229,24 @@ public void run() { }); Progress.start(progressTicket); Progress.switchToIndeterminate(progressTicket); - - - if(listInit!=null){ + + if (listInit != null) { this.foaf.addAll(listInit); } - if(actor!=null){ - process(actor, isDeepSearch,Optional.empty()); + if (actor != null) { + process(actor, isDeepSearch, Optional.empty()); } - if (listInit!=null||isDeepSearch ) { + if (listInit != null || isDeepSearch) { Progress.switchToDeterminate(progressTicket, foaf.size()); for (var foafActor : foaf) { - Progress.setDisplayName(progressTicket, "[Bsky] fetching "+actor+" n+1 > "+foafActor); - if(getIsLimitCrawlActive()){ - process(foafActor, false,Optional.of(getLimitCrawl())); + Progress.setDisplayName(progressTicket, "[Bsky] fetching " + actor + " n+1 > " + foafActor); + if (getIsLimitCrawlActive()) { + process(foafActor, false, Optional.of(getLimitCrawl())); } else { - process(foafActor, false,Optional.empty()); + process(foafActor, false, Optional.empty()); } Progress.progress(progressTicket); - + } } Progress.finish(progressTicket); @@ -256,31 +255,41 @@ public void run() { t.start(); } + private Stream manageList(String listId) { - List list = client.appBskyGraphGetList(listId); - return list.stream().flatMap(x->x.getItems().stream().map(y->y.getSubject().getDid())); - + List list = client.appBskyGraphGetList(listId); + return list.stream().flatMap(x -> x.getItems().stream().map(y -> y.getSubject().getDid())); + + } + + private void initGraphTable() { + // Create necessary model for the graph entities + if (!graphModel.getNodeTable().hasColumn("Description")) { + graphModel.getNodeTable().addColumn("Description", String.class); + } } + public void fetchFollowerFollowsFromActors(List actors, boolean isFollowsActive, boolean isFollowersActive, boolean isBlocksActive) { graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); - actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor,null, isFollowsActive, isFollowersActive, getIsDeepSearch())); + initGraphTable(); + actors.stream().forEach(actor -> fetchFollowerFollowsFromActor(actor, null, isFollowsActive, isFollowersActive, getIsDeepSearch())); } - public void fetchFollowerFollowsFromActors(List actors) { graphModel = Lookup.getDefault().lookup(GraphController.class).getGraphModel(); - actors - .stream() - .filter(x -> !x.contains("app.bsky.graph.list")) - .forEach(actor -> fetchFollowerFollowsFromActor(actor,null, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); - + initGraphTable(); + actors + .stream() + .filter(x -> !x.contains("app.bsky.graph.list")) + .forEach(actor -> fetchFollowerFollowsFromActor(actor, null, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch())); + List listActor = actors .stream() .filter(x -> x.contains("app.bsky.graph.list")) .flatMap(this::manageList) .collect(Collectors.toList()); - - fetchFollowerFollowsFromActor(null,listActor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch()); - + + fetchFollowerFollowsFromActor(null, listActor, getIsFollowsActive(), getIsFollowersActive(), getIsDeepSearch()); + } }