Complete step two of DB refactor - parallelization

This refactors the app to actually use the parallelized DB support.

Unfortunately, it doesn't actually work due to issues with the external
batch support for NamedPreparedStatement.

The next step is that, once NamedPreparedStatement has been reworked to
actually have a working version of the feature, to adapt this to use
that

2 files changed, 180 insertions, 183 deletions

diff --git a/firmal/src/main/java/bjc/firmal/Firmal.java b/firmal/src/main/java/bjc/firmal/Firmal.java
index 9e9659f..29f0fca 100644
--- a/firmal/src/main/java/bjc/firmal/Firmal.java
+++ b/firmal/src/main/java/bjc/firmal/Firmal.java
@@ -12,6 +12,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 import javax.swing.JButton;
+import javax.swing.JDesktopPane;
 import javax.swing.JDialog;
 import javax.swing.JFrame;
 import javax.swing.JInternalFrame;
@@ -27,9 +28,9 @@ import javax.swing.UIManager;
 import javax.swing.UnsupportedLookAndFeelException;
 
 import bjc.firmal.gptbrowser.GPTJSONBrowserFrame;
-import bjc.functypes.ClosableFunction;
 import bjc.functypes.ClosableThrowFunction;
 import bjc.utils.gui.layout.VLayout;
+import bjc.utils.gui.panels.BatchTaskProgressPanel;
 import bjc.utils.gui.panels.SimpleInputPanel;
 import bjc.utils.misc.BoundPreparedStatement;
 import bjc.utils.misc.NamedPreparedStatement;
@@ -60,6 +61,7 @@ public class Firmal {
 	 * The public instance of the application
 	 */
 	public static final Firmal fm = new Firmal();
+	private BatchTaskProgressPanel taskPanel = new BatchTaskProgressPanel();
 	
 	/**
 	 * General main method.
@@ -73,6 +75,7 @@ public class Firmal {
 	 * Create a new Firmal instance
 	 */
 	public Firmal() {
+		
 		dbExecutor = Executors.newSingleThreadExecutor();
 	}
 	
@@ -86,7 +89,7 @@ public class Firmal {
 	 * @return A function that will execute batches using {@link BoundPreparedStatement} to hold the data
 	 * @throws SQLException if something went wrong
 	 */
-	public SharedDBUpdateFunction createQueuedUpdater(String sql) throws SQLException {
+	public ClosableThrowFunction<BoundPreparedStatement, Future<List<Integer>>, SQLException> createQueuedUpdater(String sql) throws SQLException {
 			if (dbConnection == null) {
 				// Establish connection - we close it elsewhere
 				dbConnection = DriverManager.getConnection(connectURL, connectUser, connectPassword);
@@ -100,7 +103,17 @@ public class Firmal {
 				return result;
 			}, nameStatement);
 			
-			return (SharedDBUpdateFunction) func;
+			return func;
+	}
+	
+	/**
+	 * Create a task batch that will be tracked in the activity window
+	 * 
+	 * @param batchTitle The title for the task batch
+	 * @return The handle for that batch
+	 */
+	public BatchTaskProgressPanel.BatchHandle createTaskBatch(String batchTitle) {
+		return taskPanel.startBatch(batchTitle);
 	}
 	
 	private void buildGUI() {
@@ -118,9 +131,12 @@ public class Firmal {
 		
 		JFrame mainFrame = new JFrame();
 		
+		JDesktopPane desktop = new JDesktopPane();
+		mainFrame.setContentPane(desktop);
+		
 		mainFrame.setTitle("Firmal");
 		mainFrame.addWindowStateListener((wev) -> {
-			if (wev.getID() == WindowEvent.WINDOW_CLOSING) {
+			if (wev.getID() == WindowEvent.WINDOW_CLOSING || wev.getID() == WindowEvent.WINDOW_CLOSED) {
 				if (dbConnection != null)
 					try {
 						dbConnection.close();
@@ -135,24 +151,44 @@ public class Firmal {
 						e.printStackTrace();
 					}
 				}
-				mainFrame.dispose();
-				// TODO verify if we need to call System.exit() here, or if disposing our only window is enough
+				
+				System.exit(0);
 			}
 		});
 
 		JMenuBar mainMenuBar = new JMenuBar();
 		
+		JMenu firmalMenu = new JMenu("Firmal");
+		JMenuItem showActivityWindow = new JMenuItem("Show Activity Window");
+		showActivityWindow.addActionListener((aev) -> {
+			JInternalFrame frame = new JInternalFrame("Firmal Activity");
+			
+			frame.add(BorderLayout.CENTER, taskPanel);
+			
+			frame.setSize(480, 640);
+			frame.setVisible(true);
+			
+			frame.setClosable(true);
+			frame.setResizable(true);
+			frame.setMaximizable(true);
+			frame.setIconifiable(true);
+			
+			desktop.add(frame);
+		});
+		
+		firmalMenu.add(showActivityWindow);
+		
 		JMenu galleriaMenu = new JMenu("Galleria");
 		JMenuItem newGalleriaWindow = new JMenuItem("New...");
 		newGalleriaWindow.addActionListener((aev) -> {
-			makeGalleriaPane(mainFrame);
+			makeGalleriaPane(desktop);
 		});
 		galleriaMenu.add(newGalleriaWindow);
 		
 		JMenu chatGPTMenu = new JMenu("ChatGPT Messages");
 		JMenuItem newChatGPTMessageWindow = new JMenuItem("New...");
 		newChatGPTMessageWindow.addActionListener((aev) -> {
-			makeChatGPTPane(mainFrame);
+			makeChatGPTPane(desktop);
 		});
 		chatGPTMenu.add(newChatGPTMessageWindow);
 		
@@ -167,23 +203,23 @@ public class Firmal {
 		});
 		settingsMenu.add(dbConnection);
 		
+		mainMenuBar.add(firmalMenu);
 		mainMenuBar.add(galleriaMenu);
 		mainMenuBar.add(chatGPTMenu);
 		mainMenuBar.add(settingsMenu);
 		mainFrame.setJMenuBar(mainMenuBar);
 		
-		makeChatGPTPane(mainFrame);
+		makeChatGPTPane(desktop);
 
 		mainFrame.pack();
-		mainFrame.setSize(640, 480);
+		mainFrame.setSize(840, 680);
 		
 		mainFrame.setVisible(true);
 	}
 
-	private static void makeGalleriaPane(JFrame mainFrame) {
+	private static void makeGalleriaPane(JDesktopPane deskPane) {
 		JInternalFrame frame = GalleriaFrame.createGalleriaPane();
-		frame.setSize(840, 680);
-		frame.pack();
+		frame.setSize(640, 480);
 		frame.setVisible(true);
 		
 		frame.setClosable(true);
@@ -191,13 +227,12 @@ public class Firmal {
 		frame.setMaximizable(true);
 		frame.setIconifiable(true);
 		
-		mainFrame.add(frame);
+		deskPane.add(frame);
 	}
 	
-	private static void makeChatGPTPane(JFrame mainFrame) {
-		JInternalFrame frame = GPTJSONBrowserFrame.makeGPTJSONBrowserFrame(mainFrame);
-		frame.setSize(840, 680);
-		frame.pack();
+	private static void makeChatGPTPane(JDesktopPane deskPane) {
+		JInternalFrame frame = GPTJSONBrowserFrame.makeGPTJSONBrowserFrame(deskPane);
+		frame.setSize(640, 480);
 		frame.setVisible(true);
 
 		frame.setClosable(true);
@@ -205,7 +240,7 @@ public class Firmal {
 		frame.setMaximizable(true);
 		frame.setIconifiable(true);
 		
-		mainFrame.add(frame);
+		deskPane.add(frame);
 	}
 	
 	private JDialog prepareDBSettingsDialog(JFrame parentFrame) {
diff --git a/firmal/src/main/java/bjc/firmal/gptbrowser/GPTJSONBrowserFrame.java b/firmal/src/main/java/bjc/firmal/gptbrowser/GPTJSONBrowserFrame.java
index 64bccf0..39c98ca 100644
--- a/firmal/src/main/java/bjc/firmal/gptbrowser/GPTJSONBrowserFrame.java
+++ b/firmal/src/main/java/bjc/firmal/gptbrowser/GPTJSONBrowserFrame.java
@@ -1,19 +1,14 @@
 package bjc.firmal.gptbrowser;
 
 import java.awt.BorderLayout;
-import java.awt.Component;
 import java.awt.Dialog.ModalityType;
 import java.awt.event.ActionEvent;
 import java.awt.event.ActionListener;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.sql.Connection;
-import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
@@ -21,6 +16,7 @@ import java.util.concurrent.Future;
 
 import javax.swing.DefaultListModel;
 import javax.swing.JButton;
+import javax.swing.JDesktopPane;
 import javax.swing.JDialog;
 import javax.swing.JFileChooser;
 import javax.swing.JFrame;
@@ -30,17 +26,11 @@ import javax.swing.JList;
 import javax.swing.JMenu;
 import javax.swing.JMenuBar;
 import javax.swing.JMenuItem;
-import javax.swing.JOptionPane;
 import javax.swing.JPanel;
-import javax.swing.JPasswordField;
 import javax.swing.JProgressBar;
-import javax.swing.JScrollBar;
 import javax.swing.JScrollPane;
 import javax.swing.JSplitPane;
-import javax.swing.JTextArea;
-import javax.swing.JTextField;
 import javax.swing.JTextPane;
-import javax.swing.ListCellRenderer;
 import javax.swing.ListSelectionModel;
 import javax.swing.SwingWorker;
 
@@ -49,196 +39,167 @@ import org.json.JSONTokener;
 
 import bjc.firmal.Firmal;
 import bjc.firmal.SharedDBUpdateFunction;
+import bjc.functypes.ClosableThrowFunction;
 import bjc.utils.gui.DelegateListCellRenderer;
-import bjc.utils.gui.SimpleJList;
-import bjc.utils.gui.layout.HLayout;
 import bjc.utils.gui.layout.VLayout;
-import bjc.utils.gui.panels.SimpleInputPanel;
-import bjc.utils.ioutils.TextAreaOutputStream;
+import bjc.utils.gui.panels.BatchTaskProgressPanel.BatchHandle;
 import bjc.utils.misc.BoundPreparedStatement;
-import bjc.utils.misc.NamedPreparedStatement;
-
 import org.json.JSONArray;
 
+/**
+ * UI frame for the GPT browser
+ */
 public class GPTJSONBrowserFrame {
+	private final static int BATCH_THRESHOLD = 500;
+	
 	// Conversation data
 	private JList<GPTConversationDB> conversationListUI;
 	private DefaultListModel<GPTConversationDB> conversationListModel;
 	
-	private final class SaveConversationTask extends SwingWorker<List<Integer>, Integer> {
+	/**
+	 * Worker task to save a conversation to the DB
+	 */
+	private final class SaveConversationTask extends SwingWorker<Void, Integer> {
+		private GPTConversationDB conversation;
+		private ClosableThrowFunction<BoundPreparedStatement, Future<List<Integer>>, SQLException> insertConvFunc;
+		private ClosableThrowFunction<BoundPreparedStatement, Future<List<Integer>>, SQLException> insertMessageFunc;
 		
+		private String note;
+		
+		/**
+		 * Create a DB save worker task
+		 * @param conv The conversation to save
+		 * @param convUpdate The DB function for conversation updates
+		 * @param msgUpdate The DB function for message updates
+		 */
+		public SaveConversationTask(GPTConversationDB conv,
+				ClosableThrowFunction<BoundPreparedStatement, Future<List<Integer>>, SQLException> convUpdate,
+				ClosableThrowFunction<BoundPreparedStatement, Future<List<Integer>>, SQLException> msgUpdate) {
+			this.conversation = conv;
+			this.insertConvFunc = convUpdate;
+			this.insertMessageFunc = msgUpdate;
+			
+		}
 		@Override
-		protected List<Integer> doInBackground() throws Exception {
+		protected Void doInBackground() throws Exception {
+			BoundPreparedStatement insertConvRecord = new BoundPreparedStatement();
+			BoundPreparedStatement insertMessageRecord = new BoundPreparedStatement();
+			
+			insertConvRecord.setString("id", conversation.getID());
+			insertConvRecord.setString("title", conversation.getTitle());
+			insertConvRecord.addBatch();
+
+			List<RawMessageDB> messages = conversation.getMessages();
+			int totalNumMessages = messages.size();
+			int currNumMessages = 0;
+			
+			for (RawMessageDB message : messages) {
+                if (isCancelled()) break;
+                
+                int currProgress = Math.min(100, (currNumMessages / totalNumMessages) * 100);
+				setProgress(currProgress);
+				
+				String parentID = message.getParentMessageID();
+				if (conversation.hasSeenMessage(parentID)) {
+					String newNote = "Saving message " + currNumMessages + " of " + totalNumMessages;
+					firePropertyChange("note", note, newNote);
+					note = newNote;
+					
+					
+					insertMessageRecord.setString("selfid", message.getMessageID());
+					insertMessageRecord.setString("convid", message.getConversationID());
+					insertMessageRecord.setString("body", message.getMessageBody());
+					// TODO figure out why we are getting constraint violations here.
+					// Do we really need to leave this null initially, then backfill it?
+					// Or do we need to be doing these as independent DB queries instead of batching them?
+					// That sounds rather inefficient, but so is doing a second pass to fill it later
+					// insertMessage.setString("parentid", message.getParentMessageID());
+					insertMessageRecord.addBatch();
+				} else {
+					// TODO: this message has a missing/incorrect parent link
+				}
+			}
+			
+			String newNote = "Starting save of " + BATCH_THRESHOLD + " messages/conversations to the DB";
+			firePropertyChange("note", note, newNote);
+			note = newNote;
+			
+			// Save our changes for this conversation
+			Future<List<Integer>> insertConversationResults = insertConvFunc.apply(insertConvRecord);
+			
+			for (int i : insertConversationResults.get()) {
+				if (i != 0 && i != 1) {
+					// TODO: do something about an oddity
+				}
+			}
+
+			// TODO this isn't working right, figure out why
+			// Moved it here thinking it might have been a concurrency thing, but...
+			Future<List<Integer>> insertMessageResults = insertMessageFunc.apply(insertMessageRecord);
+			for (int i : insertMessageResults.get()) {
+				if (i != 0 && i != 1) {
+					// TODO handle oddities
+				}
+			}
+			
+			// Reset batch records
+			insertConvRecord = new BoundPreparedStatement();
+			insertMessageRecord = new BoundPreparedStatement();
+			
+			newNote = "Saved " + BATCH_THRESHOLD + " messages/conversations to the DB";
+			firePropertyChange("note", note, newNote);
+			
+			
 			return null;
 		}
 		
+		public String getNote() {
+			return note;
+		}
+		public void setNote(String note) {
+			this.note = note;
+		}
 	}
 	
 	private final class SaveConversationToDBListener implements ActionListener {
-		private final JFrame parentFrame;
+		private final JDesktopPane deskPane;
 
-		private SaveConversationToDBListener(JFrame parentFrame) {
-			this.parentFrame = parentFrame;
+		private SaveConversationToDBListener(JDesktopPane deskPane) {
+			this.deskPane = deskPane;
 		}
 
 		@Override
 		public void actionPerformed(ActionEvent aev) {
 			Firmal fm = Firmal.fm;
-			try (SharedDBUpdateFunction insertConversation = fm.createQueuedUpdater(
+			try (var insertConversation = fm.createQueuedUpdater(
 					"insert into chatgpt.conversations (conversation_id, conversation_title)"
 					+ " values (:id::uuid, :title) on conflict (conversation_id) do nothing");
-					SharedDBUpdateFunction insertMessage = fm.createQueuedUpdater(
+					var insertMessage = fm.createQueuedUpdater(
 							"insert into chatgpt.raw_messages (message_id, conversation_id, message_body) "
 							+ "values (:selfid::uuid, :convid::uuid, :body::json)"
 							+ " on conflict (message_id) do nothing")) {
 				
 				Iterator<GPTConversationDB> conversations = conversationListModel.elements().asIterator();
-				int totalNumConversations = conversationListModel.getSize();
-				int currNumConversations = 0;
-				
-				JDialog progressDialog = new JDialog(parentFrame, "DB Save Progress");
-				
-				JLabel conversationProgressLabel = new JLabel("Saving Conversation ");
-				JProgressBar conversationProgressBar = new JProgressBar();
-				conversationProgressBar.setMaximum(totalNumConversations);
-				conversationProgressBar.setValue(currNumConversations);
-				
-				JPanel conversationProgressPanel = new JPanel();
-				conversationProgressPanel.add(BorderLayout.CENTER, conversationProgressLabel);
-				conversationProgressPanel.add(BorderLayout.PAGE_END, conversationProgressBar);
-				
-				JLabel messageProgressLabel = new JLabel("Saving message: ");
-				JProgressBar messageProgressBar = new JProgressBar();
 
-				JPanel messageProgressPanel = new JPanel();
-				messageProgressPanel.add(BorderLayout.CENTER, messageProgressLabel);
-				messageProgressPanel.add(BorderLayout.PAGE_END, messageProgressBar);
-
-				JPanel contentPanel = new JPanel();
-				contentPanel.setLayout(new VLayout(2));
-				contentPanel.add(conversationProgressPanel);
-				contentPanel.add(messageProgressPanel);
-				
-				JButton okButton = new JButton("OK");
-				okButton.setEnabled(false);
-				okButton.addActionListener((aev2) -> {
-					progressDialog.dispose();
-				});
-				
-				progressDialog.add(BorderLayout.CENTER, contentPanel);
-				progressDialog.add(BorderLayout.PAGE_END, okButton);
-				
-				progressDialog.pack();
-				progressDialog.setModalityType(ModalityType.MODELESS);
-//				progressDialog.setVisible(true);
-				
-				// NOTE: Consider if this should be a text-area thing instead of progress-bar based
-				
-				// TODO: Figure out why the dialog isn't updating correctly. I probably
-				// need to use the Swing ExecutorService and split it into tasks to get things to show up properly,
-				// even if that might cause it to be a bit slower
+				BatchHandle saveBatch = fm.createTaskBatch("Save Raw ChatGPT Conversations to DB");
 				
-				// DEBUG - Check if this is actually needed
-				int overallMessageCount = 0;
-				final int BATCH_THRESHOLD = 500;
-				
-				BoundPreparedStatement insertConvRecord = new BoundPreparedStatement();
-				BoundPreparedStatement insertMessageRecord = new BoundPreparedStatement();
+				int totalConversations = conversationListModel.getSize();
+				int currConversation = 0;
 				
 				while (conversations.hasNext()) {
-					currNumConversations++;
+					currConversation++;
 					
 					GPTConversationDB conversation = conversations.next();
 
-					conversationProgressBar.setValue(currNumConversations);
-					conversationProgressLabel.setText("Saving Conversation "
-						+ currNumConversations + ": " + conversation.getTitle());
-					// DEBUG
-					System.out.println("Saving Conversation "
-						+ currNumConversations + ": " + conversation.getTitle());
-					
-					insertConvRecord.setString("id", conversation.getID());
-					insertConvRecord.setString("title", conversation.getTitle());
-					insertConvRecord.addBatch();
-
-					List<RawMessageDB> messages = conversation.getMessages();
-					int totalNumMessages = messages.size();
-					int currNumMessages = 0;
+					SaveConversationTask saveTask = new SaveConversationTask(conversation, insertConversation, insertMessage);
+					String taskDesc = "Saving conversation " + currConversation + " of " + totalConversations + ": " + conversation.getTitle();
+					saveBatch.monitorSwingWorker(saveTask, taskDesc, true);
 					
-					messageProgressBar.setMaximum(totalNumMessages);
-					for (RawMessageDB message : messages) {
-						String parentID = message.getParentMessageID();
-						if (conversation.hasSeenMessage(parentID)) {
-							messageProgressBar.setValue(++currNumMessages);
-							overallMessageCount++;
-							messageProgressLabel.setText("Saving message " + currNumMessages + " of " + totalNumMessages);
-							// DEBUG
-							System.out.println("Saving message " + currNumMessages + " of " + totalNumMessages);
-							
-							insertMessageRecord.setString("selfid", message.getMessageID());
-							insertMessageRecord.setString("convid", message.getConversationID());
-							insertMessageRecord.setString("body", message.getMessageBody());
-							// TODO figure out why we are getting constraint violations here.
-							// Do we really need to leave this null initially, then backfill it?
-							// Or do we need to be doing these as independent DB queries instead of batching them?
-							// That sounds rather inefficient, but so is doing a second pass to fill it later
-							// insertMessage.setString("parentid", message.getParentMessageID());
-							insertMessageRecord.addBatch();
-							
-							if (overallMessageCount == BATCH_THRESHOLD) {
-								overallMessageCount = 0;
-
-								// DEBUG
-								System.out.println("Starting save of " + BATCH_THRESHOLD + " messages/conversations to the DB");
-								
-								// Commit what we have so far to prevent overloading.
-								Future<List<Integer>> insertConversationResults = insertConversation.apply(insertConvRecord);
-								Future<List<Integer>> insertMessageResults = insertMessage.apply(insertMessageRecord);
-								
-								for (int i : insertConversationResults.get()) {
-									if (i != 0 && i != 1) {
-										// TODO: do something about an oddity
-									}
-								}
-								
-								for (int i : insertMessageResults.get()) {
-									if (i != 0 && i != 1) {
-										// TODO: do something about an oddity
-									}
-								}
-								
-								// Reset batch records
-								insertConvRecord = new BoundPreparedStatement();
-								insertMessageRecord = new BoundPreparedStatement();
-								
-								// DEBUG
-								System.out.println("Saved " + BATCH_THRESHOLD + " messages/conversations to the DB");
-							}
-						} else {
-							// TODO: this message has a missing/incorrect parent link
-						}
-					}
+					saveTask.execute();
 				}
-				
-				// Prepare to save things
-				Future<List<Integer>> insertConversationResults = insertConversation.apply(insertConvRecord);
-				Future<List<Integer>> insertMessageResults = insertMessage.apply(insertMessageRecord);
-				
-				for (int i : insertConversationResults.get()) {
-					if (i != 0 && i != 1) {
-						// TODO: do something about an oddity
-					}
-				}
-				
-				for (int i : insertMessageResults.get()) {
-					if (i != 0 && i != 1) {
-						// TODO: do something about an oddity
-					}
-				}
-				
-				okButton.setEnabled(true);
 			} catch (SQLException sqlex) {
-				JDialog errorDialog = new JDialog(parentFrame, "Error interfacing with DB");
+				JFrame mainFrame = null;
+				JDialog errorDialog = new JDialog(mainFrame, "Error interfacing with DB");
 				
 				JLabel headerLabel = new JLabel("Error interfacing with database");
 				
@@ -268,6 +229,7 @@ public class GPTJSONBrowserFrame {
 		}
 	}
 
+	// TODO update this to use the new SwingWorker / BatchTaskProgressPanel infrastructure
 	private static final class LoadGPTJSONListener implements ActionListener {
 		public static enum ParseMode {
 			/** Capture the whole JSON - don't parse it further. */
@@ -449,13 +411,13 @@ public class GPTJSONBrowserFrame {
 		conversationListUI.setCellRenderer(new DelegateListCellRenderer<GPTConversationDB>(GPTConversationDB::getTitle));
 	}
 	
-	public static JInternalFrame makeGPTJSONBrowserFrame(JFrame parentFrame) {
+	public static JInternalFrame makeGPTJSONBrowserFrame(JDesktopPane deskPane) {
 		GPTJSONBrowserFrame frame = new GPTJSONBrowserFrame();
 		
-		return frame.makeUIFrame(parentFrame);
+		return frame.makeUIFrame(deskPane);
 	}
 
-	private JInternalFrame makeUIFrame(JFrame parentFrame) {
+	private JInternalFrame makeUIFrame(JDesktopPane deskPane) {
 		JInternalFrame newFrame = new JInternalFrame("Conversation Browser");
 
 		JMenuBar browserMenuBar = new JMenuBar();
@@ -470,7 +432,7 @@ public class GPTJSONBrowserFrame {
 		JMenu fileMenu = new JMenu("File");
 		
 		JMenuItem saveConversations = new JMenuItem("Save Conversations...");
-		saveConversations.addActionListener(new SaveConversationToDBListener(parentFrame));
+		saveConversations.addActionListener(new SaveConversationToDBListener(deskPane));
 		fileMenu.add(saveConversations);
 		
 		JMenuItem loadConversations = new JMenuItem("Load Conversations...");