Update app.py

app.py

@@ -1,286 +1,286 @@
-import dash
-from dash import dcc, html, Input, Output, State, ctx
-import dash_bootstrap_components as dbc
-import plotly.express as px
-import pandas as pd
-import numpy as np
-import umap
-import hdbscan
-import sklearn.feature_extraction.text as text
-from dash.exceptions import PreventUpdate
-import os
-from dotenv import load_dotenv
-import helpers
-import lancedb
-from omeka_s_api_client import OmekaSClient, OmekaSClientError
-from lancedb_client import LanceDBManager
-
-# Load .env for credentials
-load_dotenv()
-_DEFAULT_PARSE_METADATA = (
-    'dcterms:identifier','dcterms:type','dcterms:title', 'dcterms:description',
-    'dcterms:creator','dcterms:publisher','dcterms:date','dcterms:spatial',
-    'dcterms:format','dcterms:provenance','dcterms:subject','dcterms:medium',
-    'bibo:annotates','bibo:content', 'bibo:locator', 'bibo:owner'
-)
-
-app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
-app.config.suppress_callback_exceptions = True
-server = app.server
-manager = LanceDBManager()
-
-french_stopwords = text.ENGLISH_STOP_WORDS.union([
-    "alors", "au", "aucuns", "aussi", "autre", "avant", "avec", "avoir", "bon",
-    "car", "ce", "cela", "ces", "ceux", "chaque", "ci", "comme", "comment", "dans",
-    "des", "du", "dedans", "dehors", "depuis", "devrait", "doit", "donc", "dos",
-    "début", "elle", "elles", "en", "encore", "essai", "est", "et", "eu", "fait",
-    "faites", "fois", "font", "hors", "ici", "il", "ils", "je", "juste", "la", "le",
-    "les", "leur", "là", "ma", "maintenant", "mais", "mes", "mine", "moins", "mon",
-    "mot", "même", "ni", "nommés", "notre", "nous", "nouveaux", "ou", "où", "par",
-    "parce", "parole", "pas", "personnes", "peut", "peu", "pièce", "plupart", "pour",
-    "pourquoi", "quand", "que", "quel", "quelle", "quelles", "quels", "qui", "sa",
-    "sans", "ses", "seulement", "si", "sien", "son", "sont", "sous", "soyez", "sujet",
-    "sur", "ta", "tandis", "tellement", "tels", "tes", "ton", "tous", "tout", "trop",
-    "très", "tu", "valeur", "voie", "voient", "vont", "votre", "vous", "vu", "ça",
-    "étaient", "état", "étions", "été", "être"
-])
-
-# -------------------- Layout --------------------
-app.layout = dbc.Container([
-    html.H2("🌍 Omeka S UMAP Explorer", className="text-center mt-4"),
-    html.Hr(),
-
-    # Input controls
-    dbc.Row([
-        dbc.Col([
-            html.H5("🔍 From Omeka S"),
-            dcc.Input(id="api-url", value="https://your-omeka-instance.org", type="text", className="form-control"),
-            dbc.Button("Load Item Sets", id="load-sets", color="secondary", className="mt-2"),
-            dcc.Dropdown(id="items-sets-dropdown", placeholder="Select a collection"),
-            dcc.Input(id="table-name", value="my_table", type="text", className="form-control mt-2", placeholder="New table name"),
-            dbc.Button("Process Omeka Collection", id="load-data", color="primary", className="mt-2"),
-        ], md=4),
-
-        dbc.Col([
-            html.H5("📁 From LanceDB"),
-            dbc.Button("Load Existing Tables", id="load-tables", color="info"),
-            dcc.Dropdown(id="db-tables-dropdown", placeholder="Select an existing table"),
-            dbc.Button("Display Table", id="load-data-db", color="success", className="mt-2"),
-        ], md=4),
-
-        dbc.Col([
-            html.H5("🔎 Query Tool (coming soon)"),
-            dbc.Input(placeholder="Type a search query...", type="text", disabled=True),
-        ], md=4),
-    ], className="mb-4"),
-
-    # Main plot area and metadata side panel
-    dbc.Row([
-        dbc.Col(
-            dcc.Graph(id="umap-graph", style={"height": "700px"}),
-            md=8
-        ),
-        dbc.Col(
-            html.Div(id="point-details", style={
-                "padding": "15px",
-                "borderLeft": "1px solid #ccc",
-                "height": "700px",
-                "overflowY": "auto"
-            }),
-            md=4
-        ),
-    ]),
-
-    # Status/info
-    html.Div(id="status", className="mt-3"),
-
-    dcc.Store(id="omeka-client-config", storage_type="session")
-], fluid=True)
-
-# -------------------- Callbacks --------------------
-
-@app.callback(
-    Output("items-sets-dropdown", "options"),
-    Output("omeka-client-config", "data"),
-    Input("load-sets", "n_clicks"),
-    State("api-url", "value"),
-    prevent_initial_call=True
-)
-def load_item_sets(n, base_url):
-    client = OmekaSClient(base_url, "...", "...", 50)
-    try:
-        item_sets = client.list_all_item_sets()
-        options = [{"label": s.get('dcterms:title', [{}])[0].get('@value', 'N/A'), "value": s["o:id"]} for s in item_sets]
-        return options, {
-            "base_url": base_url,
-            "key_identity": "...",
-            "key_credential": "...",
-            "default_per_page": 50
-        }
-    except Exception as e:
-        return dash.no_update, dash.no_update
-
-@app.callback(
-    Output("db-tables-dropdown", "options"),
-    Input("load-tables", "n_clicks"),
-    prevent_initial_call=True
-)
-def list_tables(n):
-    return [{"label": t, "value": t} for t in manager.list_tables()]
-
-@app.callback(
-    Output("umap-graph", "figure"),
-    Output("status", "children"),
-    Input("load-data", "n_clicks"),  # From Omeka S
-    Input("load-data-db", "n_clicks"),  # From DB table
-    State("items-sets-dropdown", "value"),
-    State("omeka-client-config", "data"),
-    State("table-name", "value"),
-    State("db-tables-dropdown", "value"),
-    prevent_initial_call=True
-)
-def handle_data_loading(n_clicks_omeka, n_clicks_db, item_set_id, client_config, table_name, db_table):
-    triggered_id = ctx.triggered_id
-    print(triggered_id)
-
-    if triggered_id == "load-data":  # Omeka S case
-        if not client_config:
-            raise PreventUpdate
-
-        client = OmekaSClient(
-            base_url=client_config["base_url"],
-            key_identity=client_config["key_identity"],
-            key_credential=client_config["key_credential"]
-        )
-
-        df_omeka = harvest_omeka_items(client, item_set_id=item_set_id)
-        items = df_omeka.to_dict(orient="records")
-        records_with_text = [helpers.add_concatenated_text_field_exclude_keys(item, keys_to_exclude=['id','images_urls'], text_field_key='text', pair_separator=' - ') for item in items]
-        df = helpers.prepare_df_atlas(pd.DataFrame(records_with_text), id_col='id', images_col='images_urls')
-
-        text_embed = helpers.generate_text_embed(df['text'].tolist())
-        img_embed = helpers.generate_img_embed(df['images_urls'].tolist())
-        embeddings = np.concatenate([text_embed, img_embed], axis=1)
-        df["embeddings"] = embeddings.tolist()
-
-        reducer = umap.UMAP(n_neighbors=15, min_dist=0.1, metric="cosine")
-        umap_embeddings = reducer.fit_transform(embeddings)
-        df["umap_embeddings"] = umap_embeddings.tolist()
-
-        clusterer = hdbscan.HDBSCAN(min_cluster_size=10)
-        cluster_labels = clusterer.fit_predict(umap_embeddings)
-        df["Cluster"] = cluster_labels
-
-        vectorizer = text.TfidfVectorizer(max_features=1000, stop_words=list(french_stopwords), lowercase=True)
-        tfidf_matrix = vectorizer.fit_transform(df["text"].astype(str).tolist())
-        top_words = []
-        for label in sorted(df["Cluster"].unique()):
-            if label == -1:
-                top_words.append("Noise")
-                continue
-            mask = (df["Cluster"] == label).to_numpy().nonzero()[0]
-            cluster_docs = tfidf_matrix[mask]
-            mean_tfidf = cluster_docs.mean(axis=0)
-            mean_tfidf = np.asarray(mean_tfidf).flatten()
-            top_indices = mean_tfidf.argsort()[::-1][:5]
-            terms = [vectorizer.get_feature_names_out()[i] for i in top_indices]
-            top_words.append(", ".join(terms))
-        cluster_name_map = {label: name for label, name in zip(sorted(df["Cluster"].unique()), top_words)}
-        df["Topic"] = df["Cluster"].map(cluster_name_map)
-
-        manager.initialize_table(table_name)
-        manager.add_entry(table_name, df.to_dict(orient="records"))
-
-    elif triggered_id == "load-data-db":  # Load existing LanceDB table
-        if not db_table:
-            raise PreventUpdate
-        items = manager.get_content_table(db_table)
-        df = pd.DataFrame(items)
-        df = df.dropna(axis=1, how='all')
-        df = df.fillna('')
-        #umap_embeddings = np.array(df["umap_embeddings"].tolist())
-
-    else:
-        raise PreventUpdate
-
-    # Plotting
-    return create_umap_plot(df)
-
-
-@app.callback(
-    Output("point-details", "children"),
-    Input("umap-graph", "clickData")
-)
-def show_point_details(clickData):
-    if not clickData:
-        return html.Div("🖱️ Click a point to see more details.", style={"color": "#888"})
-    img_url, title, desc = clickData["points"][0]["customdata"]
-    return html.Div([
-        html.H4(title),
-        html.Img(src=img_url, style={"maxWidth": "100%", "marginBottom": "10px"}),
-        html.P(desc or "No description available.")
-    ])
-
-# -------------------- Utility --------------------
-
-def harvest_omeka_items(client, item_set_id=None, per_page=50):
-    """
-    Fetch and parse items from Omeka S.
-    Args:
-        client: OmekaSClient instance
-        item_set_id: ID of the item set to fetch items from (optional)
-        per_page: Number of items to fetch per page (default: 50)
-    Returns:
-        DataFrame containing parsed item data
-    """
-    print("\n--- Fetching and Parsing Multiple Items by colection---")
-    try:
-        # Fetch first 5 items
-        items_list = client.list_all_items(item_set_id=item_set_id, per_page=per_page)
-        print(items_list)
-        print(f"Fetched {len(items_list)} items.")
-
-        parsed_items_list = []
-        for item_raw in items_list:
-            if 'o:media' in item_raw:
-                parsed = client.digest_item_data(item_raw, prefixes=_DEFAULT_PARSE_METADATA)
-                if parsed: # Only add if parsing was successful
-                    # Add media
-                    medias_id = [x["o:id"] for x in item_raw["o:media"]]
-                    medias_list = []
-                    for media_id in medias_id:
-                        media = client.get_media(media_id)
-                        if "image" in media["o:media_type"]:
-                          medias_list.append(media.get('o:original_url'))
-                    if medias_list: # Only append if there are image URLs
-                      parsed["images_urls"] = medias_list
-                      parsed_items_list.append(parsed)
-                      print(f"Successfully parsed {len(parsed_items_list)} items.")
-
-                print(f"Successfully parsed {len(parsed_items_list)} items.")
-        # Note: List columns (like dcterms:title) might need further handling in Pandas
-        print("\nDataFrame from parsed items:")
-        return pd.DataFrame(parsed_items_list)
-    except OmekaSClientError as e:
-        print(f"Error fetching/parsing multiple items: {e}")
-    except Exception as e:
-        print(f"An unexpected error occurred during multi-item parsing: {e}")
-        
-def create_umap_plot(df):
-    coords = np.array(df["umap_embeddings"].tolist())
-    fig = px.scatter(
-        df, x=coords[:, 0], y=coords[:, 1],
-        color="Topic",
-        custom_data=["images_urls", "Title", "Description"],
-        hover_data=None,
-        title="UMAP Projection with HDBSCAN Topics"
-    )
-    fig.update_traces(
-        marker=dict(size=8, line=dict(width=1, color="DarkSlateGrey")),
-        hovertemplate="<b>%{customdata[1]}</b><br><img src='%{customdata[0]}' height='150'><extra></extra>"
-    )
-    fig.update_layout(height=700, margin=dict(t=30, b=30, l=30, r=30))
-    return fig, f"Loaded {len(df)} items and projected into 2D."
-
-if __name__ == "__main__":
-    app.run(debug=True)
+import dash
+from dash import dcc, html, Input, Output, State, ctx
+import dash_bootstrap_components as dbc
+import plotly.express as px
+import pandas as pd
+import numpy as np
+import umap
+import hdbscan
+import sklearn.feature_extraction.text as text
+from dash.exceptions import PreventUpdate
+import os
+from dotenv import load_dotenv
+import helpers
+import lancedb
+from omeka_s_api_client import OmekaSClient, OmekaSClientError
+from lancedb_client import LanceDBManager
+
+# Load .env for credentials
+load_dotenv()
+_DEFAULT_PARSE_METADATA = (
+    'dcterms:identifier','dcterms:type','dcterms:title', 'dcterms:description',
+    'dcterms:creator','dcterms:publisher','dcterms:date','dcterms:spatial',
+    'dcterms:format','dcterms:provenance','dcterms:subject','dcterms:medium',
+    'bibo:annotates','bibo:content', 'bibo:locator', 'bibo:owner'
+)
+
+app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])
+app.config.suppress_callback_exceptions = True
+server = app.server
+manager = LanceDBManager()
+
+french_stopwords = text.ENGLISH_STOP_WORDS.union([
+    "alors", "au", "aucuns", "aussi", "autre", "avant", "avec", "avoir", "bon",
+    "car", "ce", "cela", "ces", "ceux", "chaque", "ci", "comme", "comment", "dans",
+    "des", "du", "dedans", "dehors", "depuis", "devrait", "doit", "donc", "dos",
+    "début", "elle", "elles", "en", "encore", "essai", "est", "et", "eu", "fait",
+    "faites", "fois", "font", "hors", "ici", "il", "ils", "je", "juste", "la", "le",
+    "les", "leur", "là", "ma", "maintenant", "mais", "mes", "mine", "moins", "mon",
+    "mot", "même", "ni", "nommés", "notre", "nous", "nouveaux", "ou", "où", "par",
+    "parce", "parole", "pas", "personnes", "peut", "peu", "pièce", "plupart", "pour",
+    "pourquoi", "quand", "que", "quel", "quelle", "quelles", "quels", "qui", "sa",
+    "sans", "ses", "seulement", "si", "sien", "son", "sont", "sous", "soyez", "sujet",
+    "sur", "ta", "tandis", "tellement", "tels", "tes", "ton", "tous", "tout", "trop",
+    "très", "tu", "valeur", "voie", "voient", "vont", "votre", "vous", "vu", "ça",
+    "étaient", "état", "étions", "été", "être"
+])
+
+# -------------------- Layout --------------------
+app.layout = dbc.Container([
+    html.H2("🌍 Omeka S UMAP Explorer", className="text-center mt-4"),
+    html.Hr(),
+
+    # Input controls
+    dbc.Row([
+        dbc.Col([
+            html.H5("🔍 From Omeka S"),
+            dcc.Input(id="api-url", value="https://your-omeka-instance.org", type="text", className="form-control"),
+            dbc.Button("Load Item Sets", id="load-sets", color="secondary", className="mt-2"),
+            dcc.Dropdown(id="items-sets-dropdown", placeholder="Select a collection"),
+            dcc.Input(id="table-name", value="my_table", type="text", className="form-control mt-2", placeholder="New table name"),
+            dbc.Button("Process Omeka Collection", id="load-data", color="primary", className="mt-2"),
+        ], md=4),
+
+        dbc.Col([
+            html.H5("📁 From LanceDB"),
+            dbc.Button("Load Existing Tables", id="load-tables", color="info"),
+            dcc.Dropdown(id="db-tables-dropdown", placeholder="Select an existing table"),
+            dbc.Button("Display Table", id="load-data-db", color="success", className="mt-2"),
+        ], md=4),
+
+        dbc.Col([
+            html.H5("🔎 Query Tool (coming soon)"),
+            dbc.Input(placeholder="Type a search query...", type="text", disabled=True),
+        ], md=4),
+    ], className="mb-4"),
+
+    # Main plot area and metadata side panel
+    dbc.Row([
+        dbc.Col(
+            dcc.Graph(id="umap-graph", style={"height": "700px"}),
+            md=8
+        ),
+        dbc.Col(
+            html.Div(id="point-details", style={
+                "padding": "15px",
+                "borderLeft": "1px solid #ccc",
+                "height": "700px",
+                "overflowY": "auto"
+            }),
+            md=4
+        ),
+    ]),
+
+    # Status/info
+    html.Div(id="status", className="mt-3"),
+
+    dcc.Store(id="omeka-client-config", storage_type="session")
+], fluid=True)
+
+# -------------------- Callbacks --------------------
+
+@app.callback(
+    Output("items-sets-dropdown", "options"),
+    Output("omeka-client-config", "data"),
+    Input("load-sets", "n_clicks"),
+    State("api-url", "value"),
+    prevent_initial_call=True
+)
+def load_item_sets(n, base_url):
+    client = OmekaSClient(base_url, "...", "...", 50)
+    try:
+        item_sets = client.list_all_item_sets()
+        options = [{"label": s.get('dcterms:title', [{}])[0].get('@value', 'N/A'), "value": s["o:id"]} for s in item_sets]
+        return options, {
+            "base_url": base_url,
+            "key_identity": "...",
+            "key_credential": "...",
+            "default_per_page": 50
+        }
+    except Exception as e:
+        return dash.no_update, dash.no_update
+
+@app.callback(
+    Output("db-tables-dropdown", "options"),
+    Input("load-tables", "n_clicks"),
+    prevent_initial_call=True
+)
+def list_tables(n):
+    return [{"label": t, "value": t} for t in manager.list_tables()]
+
+@app.callback(
+    Output("umap-graph", "figure"),
+    Output("status", "children"),
+    Input("load-data", "n_clicks"),  # From Omeka S
+    Input("load-data-db", "n_clicks"),  # From DB table
+    State("items-sets-dropdown", "value"),
+    State("omeka-client-config", "data"),
+    State("table-name", "value"),
+    State("db-tables-dropdown", "value"),
+    prevent_initial_call=True
+)
+def handle_data_loading(n_clicks_omeka, n_clicks_db, item_set_id, client_config, table_name, db_table):
+    triggered_id = ctx.triggered_id
+    print(triggered_id)
+
+    if triggered_id == "load-data":  # Omeka S case
+        if not client_config:
+            raise PreventUpdate
+
+        client = OmekaSClient(
+            base_url=client_config["base_url"],
+            key_identity=client_config["key_identity"],
+            key_credential=client_config["key_credential"]
+        )
+
+        df_omeka = harvest_omeka_items(client, item_set_id=item_set_id)
+        items = df_omeka.to_dict(orient="records")
+        records_with_text = [helpers.add_concatenated_text_field_exclude_keys(item, keys_to_exclude=['id','images_urls'], text_field_key='text', pair_separator=' - ') for item in items]
+        df = helpers.prepare_df_atlas(pd.DataFrame(records_with_text), id_col='id', images_col='images_urls')
+
+        text_embed = helpers.generate_text_embed(df['text'].tolist())
+        img_embed = helpers.generate_img_embed(df['images_urls'].tolist())
+        embeddings = np.concatenate([text_embed, img_embed], axis=1)
+        df["embeddings"] = embeddings.tolist()
+
+        reducer = umap.UMAP(n_neighbors=15, min_dist=0.1, metric="cosine")
+        umap_embeddings = reducer.fit_transform(embeddings)
+        df["umap_embeddings"] = umap_embeddings.tolist()
+
+        clusterer = hdbscan.HDBSCAN(min_cluster_size=10)
+        cluster_labels = clusterer.fit_predict(umap_embeddings)
+        df["Cluster"] = cluster_labels
+
+        vectorizer = text.TfidfVectorizer(max_features=1000, stop_words=list(french_stopwords), lowercase=True)
+        tfidf_matrix = vectorizer.fit_transform(df["text"].astype(str).tolist())
+        top_words = []
+        for label in sorted(df["Cluster"].unique()):
+            if label == -1:
+                top_words.append("Noise")
+                continue
+            mask = (df["Cluster"] == label).to_numpy().nonzero()[0]
+            cluster_docs = tfidf_matrix[mask]
+            mean_tfidf = cluster_docs.mean(axis=0)
+            mean_tfidf = np.asarray(mean_tfidf).flatten()
+            top_indices = mean_tfidf.argsort()[::-1][:5]
+            terms = [vectorizer.get_feature_names_out()[i] for i in top_indices]
+            top_words.append(", ".join(terms))
+        cluster_name_map = {label: name for label, name in zip(sorted(df["Cluster"].unique()), top_words)}
+        df["Topic"] = df["Cluster"].map(cluster_name_map)
+
+        manager.initialize_table(table_name)
+        manager.add_entry(table_name, df.to_dict(orient="records"))
+
+    elif triggered_id == "load-data-db":  # Load existing LanceDB table
+        if not db_table:
+            raise PreventUpdate
+        items = manager.get_content_table(db_table)
+        df = pd.DataFrame(items)
+        df = df.dropna(axis=1, how='all')
+        df = df.fillna('')
+        #umap_embeddings = np.array(df["umap_embeddings"].tolist())
+
+    else:
+        raise PreventUpdate
+
+    # Plotting
+    return create_umap_plot(df)
+
+
+@app.callback(
+    Output("point-details", "children"),
+    Input("umap-graph", "clickData")
+)
+def show_point_details(clickData):
+    if not clickData:
+        return html.Div("🖱️ Click a point to see more details.", style={"color": "#888"})
+    img_url, title, desc = clickData["points"][0]["customdata"]
+    return html.Div([
+        html.H4(title),
+        html.Img(src=img_url, style={"maxWidth": "100%", "marginBottom": "10px"}),
+        html.P(desc or "No description available.")
+    ])
+
+# -------------------- Utility --------------------
+
+def harvest_omeka_items(client, item_set_id=None, per_page=50):
+    """
+    Fetch and parse items from Omeka S.
+    Args:
+        client: OmekaSClient instance
+        item_set_id: ID of the item set to fetch items from (optional)
+        per_page: Number of items to fetch per page (default: 50)
+    Returns:
+        DataFrame containing parsed item data
+    """
+    print("\n--- Fetching and Parsing Multiple Items by colection---")
+    try:
+        # Fetch first 5 items
+        items_list = client.list_all_items(item_set_id=item_set_id, per_page=per_page)
+        print(items_list)
+        print(f"Fetched {len(items_list)} items.")
+
+        parsed_items_list = []
+        for item_raw in items_list:
+            if 'o:media' in item_raw:
+                parsed = client.digest_item_data(item_raw, prefixes=_DEFAULT_PARSE_METADATA)
+                if parsed: # Only add if parsing was successful
+                    # Add media
+                    medias_id = [x["o:id"] for x in item_raw["o:media"]]
+                    medias_list = []
+                    for media_id in medias_id:
+                        media = client.get_media(media_id)
+                        if "image" in media["o:media_type"]:
+                          medias_list.append(media.get('o:original_url'))
+                    if medias_list: # Only append if there are image URLs
+                      parsed["images_urls"] = medias_list
+                      parsed_items_list.append(parsed)
+                      print(f"Successfully parsed {len(parsed_items_list)} items.")
+
+                print(f"Successfully parsed {len(parsed_items_list)} items.")
+        # Note: List columns (like dcterms:title) might need further handling in Pandas
+        print("\nDataFrame from parsed items:")
+        return pd.DataFrame(parsed_items_list)
+    except OmekaSClientError as e:
+        print(f"Error fetching/parsing multiple items: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred during multi-item parsing: {e}")
+        
+def create_umap_plot(df):
+    coords = np.array(df["umap_embeddings"].tolist())
+    fig = px.scatter(
+        df, x=coords[:, 0], y=coords[:, 1],
+        color="Topic",
+        custom_data=["images_urls", "Title", "Description"],
+        hover_data=None,
+        title="UMAP Projection with HDBSCAN Topics"
+    )
+    fig.update_traces(
+        marker=dict(size=8, line=dict(width=1, color="DarkSlateGrey")),
+        hovertemplate="<b>%{customdata[1]}</b><br><img src='%{customdata[0]}' height='150'><extra></extra>"
+    )
+    fig.update_layout(height=700, margin=dict(t=30, b=30, l=30, r=30))
+    return fig, f"Loaded {len(df)} items and projected into 2D."
+
+if __name__ == "__main__":
+    app.run(debug=True, port=7860)