// static/js/sidebar.js - Part 1 of 3

// Global variables
let specialtyClustersMap = {};
let specialtyHierarchy = {};
const checkboxesState = {};
let availableSpecialtiesFromAPI = [];
let selectedSpecialtyClusters = {};
let initialLoad = true;
let isSubmitting = false;
let selectedSpecialtyNames = [];

// Check for URL parameters on load to detect if we have filters
let hasUrlFilters = false;

document.addEventListener('DOMContentLoaded', function() {
    // Check if the URL has any specialty_cluster_filter or cluster_filter parameters
    const urlParams = new URLSearchParams(window.location.search);
    hasUrlFilters = urlParams.has('specialty_cluster_filter') || urlParams.has('cluster_filter');
    
    // Add article IDs as data attributes to each row to help with specialty columns
    const rows = document.querySelectorAll('table.article-list tbody tr');
    rows.forEach((row, index) => {
        // Try to extract article ID
        let articleId = 'article-' + index;
        
        // First try: from a link in the row if it has an ID parameter
        const links = row.querySelectorAll('a[href*="id="]');
        if (links.length > 0) {
            const href = links[0].getAttribute('href');
            const idMatch = href.match(/id=(\d+)/);
            if (idMatch && idMatch[1]) {
                articleId = idMatch[1];
            }
        }
        
        // Second try: from the article title content or summary (first few characters)
        if (articleId === 'article-' + index) {
            const titleElem = row.querySelector('.article-title');
            if (titleElem) {
                // Create a simple hash from the title text
                const titleText = titleElem.textContent.trim();
                articleId = 'title_' + titleText.slice(0, 10).replace(/\W+/g, '_');
            }
        }
        
        // Store the ID as a data attribute on the row
        row.setAttribute('data-article-id', articleId);
    });
    
    // Check URL parameters for column visibility
    const showIntlRatings = urlParams.has('show_intl_ratings');
    const showSpecialtyRatings = urlParams.has('show_specialty_ratings');
    
    // Set checkboxes based on URL parameters
    document.getElementById('show-intl-ratings').checked = showIntlRatings;
    document.getElementById('show-specialty-ratings').checked = showSpecialtyRatings;
    
    // Update the international ratings header to show it's using Deepseek
    const intlRatingsHeader = document.querySelector('table.article-list thead th:nth-child(5)');
    if (intlRatingsHeader) {
        const sortBy = document.getElementById('sort_by').value;
        const sortDir = document.getElementById('sort_dir').value;
        
        // Update the header text
        intlRatingsHeader.innerHTML = `
            Deepseek Rating
            ${sortBy === 'deepseek_rating' ? `<span class="sort-icon sort-${sortDir}"></span>` : ''}
        `;
        
        // Ensure clicking this header sorts by deepseek_rating
        intlRatingsHeader.setAttribute('onclick', "sortBy('deepseek_rating')");
    }
    
    fetchAndPopulateData();
    setupSourceAutocomplete();
    ensureSpecialtyNameInClusters();
    setupClusterAutocomplete();
    updateColumnVisibility();
    setupArticleListHandlers();
    
    // Add event listeners to checkboxes to update visibility
    document.getElementById('show-intl-ratings').addEventListener('change', function() {
        updateColumnVisibility();
    });
    
    document.getElementById('show-specialty-ratings').addEventListener('change', function() {
        updateColumnVisibility();
    });
    
    document.addEventListener('click', function(e) {
        if (!e.target.closest('.autocomplete-container')) {
            document.querySelectorAll('.autocomplete-dropdown').forEach(dropdown => {
                dropdown.classList.remove('active');
            });
        }
    });
    
    // Mobile sidebar toggle functionality
    const sidebarToggle = document.getElementById('sidebar-toggle');
    const sidebar = document.getElementById('sidebar');
    const sidebarOverlay = document.getElementById('sidebar-overlay');
    const sidebarClose = document.getElementById('sidebar-close');
    
    if (sidebarToggle && sidebar && sidebarOverlay) {
        sidebarToggle.addEventListener('click', function() {
            sidebar.classList.add('active');
            sidebarOverlay.classList.add('active');
        });
        
        sidebarOverlay.addEventListener('click', function() {
            sidebar.classList.remove('active');
            sidebarOverlay.classList.remove('active');
        });
        
        if (sidebarClose) {
            sidebarClose.addEventListener('click', function() {
                sidebar.classList.remove('active');
                sidebarOverlay.classList.remove('active');
            });
        }
    }
    
    initializeCollapsedSections();
    
    // Initialize selected specialty clusters state from current URL parameters
    loadSelectedClustersFromUrl();
    
    // Extract primary specialties from checked checkboxes or URL parameters
    setTimeout(() => {
        // Get the primary specialties from checked checkboxes
        const checkedSpecialtyBoxes = document.querySelectorAll('.specialty-checkbox:checked');
        const primarySpecialties = new Set();
        
        checkedSpecialtyBoxes.forEach(checkbox => {
            const label = checkbox.nextElementSibling;
            if (label) {
                const specialtyName = label.textContent.split('(')[0].trim();
                primarySpecialties.add(specialtyName);
            }
        });
        
        // Create columns only for primary specialties, not all clusters
        Array.from(primarySpecialties).forEach(specialtyName => {
            setTimeout(() => {
                updateSpecialtyColumnRatings(specialtyName);
            }, 200);
        });
    }, 500);
    
    // Check URL parameters for cluster filters to show specialty columns for specialty clusters
    const clusterFilters = urlParams.getAll('cluster_filter');
    
    // For each cluster filter, check if it's a specialty and show its column
    if (clusterFilters.length > 0) {
        const specialtyClusters = clusterFilters.filter(cluster => isSpecialtyCluster(cluster));
        
        if (specialtyClusters.length > 0) {
            setTimeout(() => {
                specialtyClusters.forEach(cluster => {
                    // Format the cluster name with proper capitalization
                    const formattedCluster = cluster.split(/\s+/)
                        .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())
                        .join(' ');
                    
                    updateSpecialtyColumnRatings(formattedCluster);
                });
            }, 500);
        }
    }
    
    // Periodically check if columns should be visible but aren't
    setInterval(() => {
        // Check specialty columns
        selectedSpecialtyNames.forEach(specialtyName => {
            const specialtyId = getSpecialtyId(specialtyName);
            const headerCell = document.getElementById(`specialty-column-${specialtyId}`);
            if (headerCell && headerCell.style.display !== 'table-cell') {
                toggleSpecialtyColumn(specialtyName, true);
            }
        });
        
        // Also check regular columns if they should be visible
        const showIntlRatings = document.getElementById('show-intl-ratings').checked;
        const showSpecialtyRatings = document.getElementById('show-specialty-ratings').checked;
        
        if (showIntlRatings || showSpecialtyRatings) {
            const table = document.querySelector('table');
            if (table) {
                const headerRow = table.querySelector('thead tr');
                const rows = table.querySelectorAll('tbody tr');
                const headerCells = headerRow.querySelectorAll('th');
                
                const intlRatingColIndex = 4;
                const specialtyRatingColIndex = 5;
                
                if (showIntlRatings && headerCells[intlRatingColIndex].style.display !== 'table-cell') {
                    headerCells[intlRatingColIndex].style.display = 'table-cell';
                    rows.forEach(row => {
                        const cells = row.querySelectorAll('td');
                        if (cells.length > intlRatingColIndex) {
                            cells[intlRatingColIndex].style.display = 'table-cell';
                        }
                    });
                }
                
                if (showSpecialtyRatings && headerCells[specialtyRatingColIndex].style.display !== 'table-cell') {
                    headerCells[specialtyRatingColIndex].style.display = 'table-cell';
                    rows.forEach(row => {
                        const cells = row.querySelectorAll('td');
                        if (cells.length > specialtyRatingColIndex) {
                            cells[specialtyRatingColIndex].style.display = 'table-cell';
                        }
                    });
                }
            }
        }
    }, 1000);
    
    initialLoad = false;
});

// KEY FUNCTION: Toggle collapsible sections
function toggleSection(element) {
    const content = element.nextElementSibling;
    const icon = element.querySelector('.toggle-icon');
    
    if (content && icon) {
        content.classList.toggle('collapsed');
        icon.textContent = content.classList.contains('collapsed') ? '▶' : '▼';
    }
}

// Function to initialize collapsed sections
function initializeCollapsedSections() {
    const filterSections = document.querySelectorAll('.filter-section');
    filterSections.forEach((section, index) => {
        const title = section.querySelector('.filter-section-title');
        const content = section.querySelector('.collapsible-content');
        const icon = title?.querySelector('.toggle-icon');
        
        // Only keep first section expanded, collapse the rest
        if (index !== 0 && content) {
            content.classList.add('collapsed');
            if (icon) icon.textContent = '▶';
        }
    });
}

// Show loading indicator
function showLoader() {
    if (isSubmitting) return;
    isSubmitting = true;
    
    const loader = document.getElementById('loader-overlay');
    loader.classList.add('active');
}

// Hide loading indicator
function hideLoader() {
    isSubmitting = false;
    const loader = document.getElementById('loader-overlay');
    loader.classList.remove('active');
}

// Function to check if a cluster is a specialty
function isSpecialtyCluster(clusterName) {
    // First, normalize the cluster name
    const normalizedCluster = clusterName.toLowerCase().trim();
    
    // Check if it's an exact match for any specialty name
    for (const specialty in specialtyClustersMap) {
        if (specialty.toLowerCase() === normalizedCluster) {
            return true;
        }
    }
    
    // Also check for common patterns that indicate specialty names
    const specialtyPatterns = [
        /^(allergy|anesthesiology|cardiology|dermatology|emergency|endocrinology|ent|family|gastroenterology|general|geriatrics|hematology|infectious|internal|interventional|neonatology|nephrology|neurology|neurophysiology|neuroradiology|neurosurgery|nurse|obstetrics|oncology|ophthalmology|orthopedic|pain|pathology|pediatric|pharmacist|physical|plastic|psychiatry|pulmonology|radiation|radiology|rheumatology|surgery|trauma|urgent|urology|vascular)/i
    ];
    
    return specialtyPatterns.some(pattern => pattern.test(normalizedCluster));
}

// Load selected specialty clusters from URL parameters
function loadSelectedClustersFromUrl() {
    selectedSpecialtyClusters = {};
    const urlParams = new URLSearchParams(window.location.search);
    
    // First check for specialty_cluster_filter
    const specialtyClusterFilters = urlParams.getAll('specialty_cluster_filter');
    if (specialtyClusterFilters.length > 0) {
        specialtyClusterFilters.forEach(cluster => {
            selectedSpecialtyClusters[cluster] = true;
        });
    }
    
    // If no specialty clusters, check for cluster_filter (to mark checkboxes)
    const clusterFilters = urlParams.getAll('cluster_filter');
    if (specialtyClusterFilters.length === 0 && clusterFilters.length > 0) {
        clusterFilters.forEach(cluster => {
            selectedSpecialtyClusters[cluster] = true;
        });
    }
}

function toggleSpecialtyColumn(specialtyName, show) {
    const specialtyId = getSpecialtyId(specialtyName);
    const headerCell = document.getElementById(`specialty-column-${specialtyId}`);
    const cellsToToggle = document.querySelectorAll(`[id^="specialty-cell-${specialtyId}-"]`);
    
    if (headerCell) {
        headerCell.style.display = show ? 'table-cell' : 'none';
    }
    
    cellsToToggle.forEach(cell => {
        cell.style.display = show ? 'table-cell' : 'none';
    });
    
    // If showing, set a timeout to ensure it stays visible
    if (show) {
        setTimeout(() => {
            if (headerCell) {
                headerCell.style.display = 'table-cell';
            }
            cellsToToggle.forEach(cell => {
                cell.style.display = 'table-cell';
            });
        }, 100);
    }
}

// Function to create a specialty column if it doesn't exist
function createSpecialtyColumn(specialtyName) {
    const specialtyId = getSpecialtyId(specialtyName);
    
    // Check if column already exists
    if (document.getElementById(`specialty-column-${specialtyId}`)) {
        return;
    }
    
    // Create header cell
    const headerRow = document.querySelector('table.article-list thead tr');
    if (headerRow) {
        const intlRatingHeader = headerRow.querySelector('th:nth-child(5)');
        if (intlRatingHeader) {
            intlRatingHeader.innerHTML = `
                Deepseek Rating
                <span class="sort-icon sort-${document.getElementById('sort_dir').value || 'desc'}"></span>
            `;
            
            // Update the onclick attribute
            intlRatingHeader.setAttribute('onclick', "sortBy('deepseek_rating')");
        }
    }
    const newHeaderCell = document.createElement('th');
    newHeaderCell.id = `specialty-column-${specialtyId}`;
    newHeaderCell.className = 'specialty-column sortable';
    newHeaderCell.style.width = '120px';
    newHeaderCell.innerHTML = `<span>${specialtyName}</span>`;
    newHeaderCell.onclick = function() { sortBySpecialty(specialtyName); };
    headerRow.appendChild(newHeaderCell);
    
    // Create cells for each article row
    const rows = document.querySelectorAll('table.article-list tbody tr');
    rows.forEach(row => {
        const articleId = row.getAttribute('data-article-id');
        
        const newCell = document.createElement('td');
        newCell.id = `specialty-cell-${specialtyId}-${articleId}`;
        newCell.className = 'specialty-column';
        newCell.innerHTML = `<span style="color: #9ca3af;">-</span>`;
        row.appendChild(newCell);
    });
}

// Helper function to get a valid ID from specialty name
function getSpecialtyId(specialtyName) {
    return specialtyName.toLowerCase().replace(/[^a-z0-9]/g, '_');
}

// Function to update a specialty column with ratings
function updateSpecialtyColumnRatings(specialtyName) {
    const specialtyId = getSpecialtyId(specialtyName);
    
    // Create the column if it doesn't exist
    createSpecialtyColumn(specialtyName);
    
    // Add to the list of selected specialties if not already there
    if (!selectedSpecialtyNames.includes(specialtyName)) {
        selectedSpecialtyNames.push(specialtyName);
    }
    
    // Get all article rows
    const rows = document.querySelectorAll('table.article-list tbody tr');
    
    rows.forEach(row => {
        const articleId = row.getAttribute('data-article-id');
        
        // Get the specialty-specific cell for this article
        const specialtyCell = row.querySelector(`#specialty-cell-${specialtyId}-${articleId}`);
        if (!specialtyCell) return;
        
        // Get the regular specialty ratings cell for this article
        const specialtyRatingsCell = row.querySelector('td:nth-child(6)');
        if (!specialtyRatingsCell) return;
        
        // Get all rating pills
        const ratingPills = specialtyRatingsCell.querySelectorAll('.rating-pill');
        let found = false;
        
        // Find the rating for this specialty
        ratingPills.forEach(pill => {
            const pillText = pill.textContent || '';
            
            // Look for exact specialty name match (e.g., "Cardiology: 8")
            if (pillText.startsWith(specialtyName + ':')) {
                found = true;
                
                // Extract just the rating number
                const ratingMatch = pillText.match(/:\s*(\d+)/);
                if (ratingMatch && ratingMatch[1]) {
                    const rating = ratingMatch[1];
                    
                    // Set the rating without the specialty name prefix
                    specialtyCell.innerHTML = `<div class="rating-pill">${rating}</div>`;
                    
                    // Store the rating as a data attribute for sorting
                    specialtyCell.setAttribute('data-rating', rating);
                }
            }
        });
        
        // If no rating found for this specialty, show a blank placeholder
        if (!found) {
            specialtyCell.innerHTML = `<span style="color: #9ca3af;">-</span>`;
            specialtyCell.setAttribute('data-rating', '0');
        }
    });
    
    // Show the column
    toggleSpecialtyColumn(specialtyName, true);
}

// Function to sort by specialty rating
function sortBySpecialty(specialtyName) {
    const specialtyId = getSpecialtyId(specialtyName);
    
    // Get the current sort direction
    let sortDir = 'desc';
    const headerCell = document.getElementById(`specialty-column-${specialtyId}`);
    if (headerCell) {
        const currentSortDir = headerCell.getAttribute('data-sort-dir');
        sortDir = currentSortDir === 'desc' ? 'asc' : 'desc';
        
        // Update the sort direction attribute
        headerCell.setAttribute('data-sort-dir', sortDir);
        
        // Remove sort indicators from all headers
        document.querySelectorAll('th.specialty-column').forEach(th => {
            th.classList.remove('sort-asc', 'sort-desc');
        });
        
        // Add sort indicator to this header
        headerCell.classList.add(`sort-${sortDir}`);
    }
    
    // Get all rows
    const tbody = document.querySelector('table.article-list tbody');
    const rows = Array.from(tbody.querySelectorAll('tr'));
    
    // Sort the rows by the specialty rating
    rows.sort((a, b) => {
        const cellA = a.querySelector(`#specialty-cell-${specialtyId}-${a.getAttribute('data-article-id')}`);
        const cellB = b.querySelector(`#specialty-cell-${specialtyId}-${b.getAttribute('data-article-id')}`);
        
        const ratingA = parseInt(cellA?.getAttribute('data-rating') || '0');
        const ratingB = parseInt(cellB?.getAttribute('data-rating') || '0');
        
        // Handle empty ratings (showing "-")
        if (isNaN(ratingA) && isNaN(ratingB)) return 0;
        if (isNaN(ratingA)) return sortDir === 'asc' ? -1 : 1;
        if (isNaN(ratingB)) return sortDir === 'asc' ? 1 : -1;
        
        // Sort by rating
        return sortDir === 'asc' 
            ? ratingA - ratingB
            : ratingB - ratingA;
    });
    
    // Reappend the rows in the new order
    rows.forEach(row => tbody.appendChild(row));
}

function clearFilters() {
    window.location.href = '/doximity_new';
}
// static/js/sidebar.js - Part 3 of 4

function searchSources(query) {
    const dropdown = document.getElementById('source-dropdown');
    if (!query || query.length < 1) {
        dropdown.classList.remove('active');
        return;
    }
    
    const filteredSources = availableSources.filter(source => 
        source.toLowerCase().includes(query.toLowerCase()) &&
        !document.querySelector(`#selected-sources .selected-item[data-value="${source}"]`)
    ).slice(0, 20);
    
    if (filteredSources.length > 0) {
        dropdown.innerHTML = '';
        filteredSources.forEach(source => {
            const item = document.createElement('div');
            item.className = 'autocomplete-item';
            item.textContent = source;
            item.onclick = function() { addSource(source); };
            dropdown.appendChild(item);
        });
        dropdown.classList.add('active');
    } else {
        dropdown.classList.remove('active');
    }
}

function searchClusters(query) {
    const dropdown = document.getElementById('cluster-dropdown');
    if (!query || query.length < 1) {
        dropdown.classList.remove('active');
        return;
    }
    
    const filteredClusters = availableClusters.filter(cluster => 
        cluster.toLowerCase().includes(query.toLowerCase()) &&
        !document.querySelector(`#selected-clusters .selected-item[data-value="${cluster}"]`)
    ).slice(0, 20);
    
    if (filteredClusters.length > 0) {
        dropdown.innerHTML = '';
        filteredClusters.forEach(cluster => {
            const item = document.createElement('div');
            item.className = 'autocomplete-item';
            item.textContent = cluster;
            item.onclick = function() { addCluster(cluster); };
            dropdown.appendChild(item);
        });
        dropdown.classList.add('active');
    } else {
        dropdown.classList.remove('active');
    }
}

function setupSourceAutocomplete() {
    const input = document.getElementById('source-input');
    if (input) input.addEventListener('input', function() { searchSources(this.value); });
}

function setupClusterAutocomplete() {
    const input = document.getElementById('cluster-input');
    if (input) input.addEventListener('input', function() { searchClusters(this.value); });
}

function addSource(source) {
    showLoader();
    
    const container = document.getElementById('selected-sources');
    if (!container || container.querySelector(`.selected-item[data-value="${source}"]`)) return;
    
    const item = document.createElement('div');
    item.className = 'selected-item';
    item.setAttribute('data-value', source);
    item.innerHTML = `
        ${source}
        <span class="remove" onclick="removeSource('${source.replace(/'/g, "\\'")}')">✕</span>
        <input type="hidden" name="source_filter" value="${source}">
    `;
    container.appendChild(item);
    
    document.getElementById('source-input').value = '';
    document.getElementById('source-dropdown').classList.remove('active');
    document.getElementById('filter-form').submit();
}

function removeSource(source) {
    showLoader();
    
    const item = document.querySelector(`#selected-sources .selected-item[data-value="${source}"]`);
    if (item) {
        item.remove();
        document.getElementById('filter-form').submit();
    }
}

function addCluster(cluster) {
    showLoader();
    
    const container = document.getElementById('selected-clusters');
    if (!container || container.querySelector(`.selected-item[data-value="${cluster}"]`)) return;
    
    const item = document.createElement('div');
    item.className = 'selected-item';
    item.setAttribute('data-value', cluster);
    item.innerHTML = `
        ${cluster}
        <span class="remove" onclick="removeCluster('${cluster.replace(/'/g, "\\'")}')">✕</span>
        <input type="hidden" name="cluster_filter" value="${cluster}">
    `;
    container.appendChild(item);
    
    document.getElementById('cluster-input').value = '';
    document.getElementById('cluster-dropdown').classList.remove('active');
    document.getElementById('filter-form').submit();
}

function removeCluster(cluster) {
    showLoader();
    
    const item = document.querySelector(`#selected-clusters .selected-item[data-value="${cluster}"]`);
    if (item) {
        item.remove();
        document.getElementById('filter-form').submit();
    }
}

function normalizeSpecialtyName(name) {
    return name ? name.toLowerCase().trim() : '';
}

function fetchAndPopulateData() {
    fetch('/api/articles/debug')
        .then(response => {
            if (!response.ok) throw new Error(`HTTP error ${response.status}`);
            return response.json();
        })
        .then(data => {
            availableSpecialtiesFromAPI = data.available_specialties || [];
            buildSpecialtyGroups(data.specialty_names || [], data.available_clusters || []);
            processSpecialtyHierarchy(availableSpecialtiesFromAPI);
            fixSpecialtyIssues();
            consolidateByHierarchy();
            matchDiagnosticSpecialties();
            renderDropdown();
        })
        .catch(error => {
            console.error("Error fetching data:", error);
            buildSpecialtyGroups([], availableClusters || []);
            renderDropdown();
        });
}
// static/js/sidebar.js - Part 4 of 4

function buildSpecialtyGroups(specialtyNames, clusters) {
    specialtyClustersMap = {};
    availableSpecialtiesFromAPI.forEach(specialty => {
        specialtyClustersMap[normalizeSpecialtyName(specialty.name)] = [];
    });

    if (clusters && Array.isArray(clusters)) {
        clusters.forEach(cluster => {
            const clusterText = String(cluster).toLowerCase();
            if (clusterText.length < 3) return;

            availableSpecialtiesFromAPI.forEach(specialty => {
                const specialtyLower = specialty.name.toLowerCase();
                const normalizedSpecialty = normalizeSpecialtyName(specialty.name);
                const patterns = [
                    specialtyLower,
                    `for ${specialtyLower}`,
                    `in ${specialtyLower}`,
                    `${specialtyLower} -`,
                    `top ${specialtyLower}`,
                    `advances in ${specialtyLower}`,
                    `journal of ${specialtyLower}`
                ];

                if (patterns.some(pattern => clusterText.includes(pattern))) {
                    if (!specialtyClustersMap[normalizedSpecialty].includes(cluster)) {
                        specialtyClustersMap[normalizedSpecialty].push(cluster);
                    }
                }
            });
        });
    }

    // Ensure each specialty includes itself as a cluster if available in the clusters array
    availableSpecialtiesFromAPI.forEach(specialty => {
        const normalizedSpecialty = normalizeSpecialtyName(specialty.name);
        // First check if specialty name itself exists in clusters array
        const specialtyNameCluster = clusters.find(cluster => 
            cluster.toLowerCase() === specialty.name.toLowerCase());
        
        if (specialtyNameCluster && 
            !specialtyClustersMap[normalizedSpecialty].includes(specialtyNameCluster)) {
            specialtyClustersMap[normalizedSpecialty].push(specialtyNameCluster);
        }
    });

    consolidateByHierarchy();
    Object.keys(specialtyClustersMap).forEach(specialty => {
        if (!specialtyClustersMap[specialty].length) delete specialtyClustersMap[specialty];
    });
}

function processSpecialtyHierarchy(availableSpecialties) {
    specialtyHierarchy = {};
    if (availableSpecialties && Array.isArray(availableSpecialties)) {
        availableSpecialties.forEach(specialty => {
            if (!specialty.parent_id) {
                const normalizedName = normalizeSpecialtyName(specialty.name);
                specialtyHierarchy[normalizedName] = {
                    id: specialty.id,
                    name: specialty.name,
                    children: [],
                    directClusters: []
                };
            }
        });

        availableSpecialties.forEach(specialty => {
            if (specialty.parent_id) {
                const parentSpecialty = availableSpecialties.find(s => s.id === specialty.parent_id);
                if (parentSpecialty) {
                    const parentName = normalizeSpecialtyName(parentSpecialty.name);
                    const childName = normalizeSpecialtyName(specialty.name);
                    if (specialtyHierarchy[parentName]) {
                        specialtyHierarchy[parentName].children.push({
                            id: specialty.id,
                            name: childName
                        });
                    }
                }
            }
        });
    }
}

function consolidateByHierarchy() {
    Object.entries(specialtyHierarchy).forEach(([parentName, parentData]) => {
        const normalizedParent = normalizeSpecialtyName(parentName);
        if (!specialtyClustersMap[normalizedParent]) specialtyClustersMap[normalizedParent] = [];
        
        parentData.children.forEach(child => {
            const childName = normalizeSpecialtyName(child.name);
            if (specialtyClustersMap[childName]) {
                specialtyClustersMap[childName].forEach(cluster => {
                    if (!specialtyClustersMap[normalizedParent].includes(cluster)) {
                        specialtyClustersMap[normalizedParent].push(cluster);
                    }
                });
            }
        });
    });
}

function fixSpecialtyIssues() {
    if (specialtyClustersMap['urology']) {
        specialtyClustersMap['urology'] = specialtyClustersMap['urology'].filter(cluster => {
            const text = cluster.toLowerCase();
            return /\burology\b/.test(text) && !/\bneurology\b/.test(text);
        });
    }
    if (specialtyClustersMap['neurology']) {
        specialtyClustersMap['neurology'] = specialtyClustersMap['neurology'].filter(cluster => {
            const text = cluster.toLowerCase();
            return /\bneurology\b/.test(text) && !/\burology\b/.test(text);
        });
    }
}

function matchDiagnosticSpecialties() {
    const diagnosticSpecialties = [
        "allergy & immunology", "anesthesiology", "cardiology", 
        "child & adolescent psychiatry", "critical care", "dermatology",
        "emergency medicine", "endocrinology", "ent", "family medicine",
        "gastroenterology", "general surgery", "geriatrics", "hematology",
        "infectious disease", "internal medicine", "interventional radiology",
        "neonatology & perinatology", "nephrology", "neurology", "neurophysiology",
        "neuroradiology", "neurosurgery", "nurse practitioner", "obstetrics & gynecology",
        "oncology", "ophthalmology", "orthopedic sports medicine", "orthopedic surgery",
        "pain medicine", "pathology", "pediatric cardiology", "pediatric emergency medicine",
        "pediatric endocrinology", "pediatric gastroenterology", "pediatric hematology & oncology",
        "pediatric infectious disease", "pediatric pulmonology", "pediatrics", "pharmacist",
        "physical medicine/rehab", "plastic surgery", "psychiatric-mental health", "psychiatry",
        "pulmonology", "radiation oncology", "radiology", "rheumatology", "thoracic surgery",
        "trauma surgery", "urgent care", "urology", "vascular surgery"
    ];
    
    const filteredMap = {};
    diagnosticSpecialties.forEach(specialty => {
        const normalizedSpecialty = normalizeSpecialtyName(specialty);
        if (specialtyClustersMap[normalizedSpecialty]) {
            filteredMap[normalizedSpecialty] = specialtyClustersMap[normalizedSpecialty];
        }
    });
    specialtyClustersMap = filteredMap;
}

// Function to update hidden inputs for specialty clusters
function updateSpecialtyClusterInputs() {
    const inputsContainer = document.getElementById('specialty-cluster-inputs');
    if (!inputsContainer) return;
    
    // First, remove all existing specialty cluster filter inputs
    document.querySelectorAll('input[name="specialty_cluster_filter"]').forEach(input => {
        input.remove();
    });
    
    // Add hidden inputs for all selected clusters
    Object.keys(selectedSpecialtyClusters).forEach(cluster => {
        const input = document.createElement('input');
        input.type = 'hidden';
        input.name = 'specialty_cluster_filter';
        input.value = cluster;
        inputsContainer.appendChild(input);
    });
}

function ensureSpecialtyNameInClusters() {
    // This function can be called during form submission
    document.getElementById('filter-form').addEventListener('submit', function(e) {
        // Get all checked specialty checkboxes
        const checkedSpecialtyBoxes = document.querySelectorAll('.specialty-checkbox:checked');
        
        checkedSpecialtyBoxes.forEach(checkbox => {
            // Get the specialty name from the label
            const specialtyLabel = checkbox.nextElementSibling.textContent;
            const specialtyName = specialtyLabel.split('(')[0].trim();
            
            // Add an additional hidden input with just the specialty name
            const input = document.createElement('input');
            input.type = 'hidden';
            input.name = 'specialty_cluster_filter';
            input.value = specialtyName;
            document.getElementById('specialty-cluster-inputs').appendChild(input);
        });
    });
}

function updateColumnVisibility() {
    const showIntlRatings = document.getElementById('show-intl-ratings').checked;
    const showSpecialtyRatings = document.getElementById('show-specialty-ratings').checked;
    
    // Store the visibility state in hidden form fields
    const form = document.getElementById('filter-form');
    
    // Remove existing visibility inputs if any
    document.querySelectorAll('input[name="show_intl_ratings"]').forEach(input => input.remove());
    document.querySelectorAll('input[name="show_specialty_ratings"]').forEach(input => input.remove());
    
    // Add new inputs with current state
    if (showIntlRatings) {
        const input = document.createElement('input');
        input.type = 'hidden';
        input.name = 'show_intl_ratings';
        input.value = '1';
        form.appendChild(input);
    }
    
    if (showSpecialtyRatings) {
        const input = document.createElement('input');
        input.type = 'hidden';
        input.name = 'show_specialty_ratings';
        input.value = '1';
        form.appendChild(input);
    }
    
    const table = document.querySelector('table');
    if (!table) return;
    
    const headerRow = table.querySelector('thead tr');
    const rows = table.querySelectorAll('tbody tr');
    
    const intlRatingColIndex = 4;
    const specialtyRatingColIndex = 5;
    
    const headerCells = headerRow.querySelectorAll('th');
    headerCells[intlRatingColIndex].style.display = showIntlRatings ? 'table-cell' : 'none';
    headerCells[specialtyRatingColIndex].style.display = showSpecialtyRatings ? 'table-cell' : 'none';
    
    rows.forEach(row => {
        const cells = row.querySelectorAll('td');
        cells[intlRatingColIndex].style.display = showIntlRatings ? 'table-cell' : 'none';
        cells[specialtyRatingColIndex].style.display = showSpecialtyRatings ? 'table-cell' : 'none';
        
        if (!showIntlRatings && !showSpecialtyRatings) {
            cells[1].style.width = '60%';
        } else if (!showIntlRatings || !showSpecialtyRatings) {
            cells[1].style.width = '50%';
        } else {
            cells[1].style.width = '40%';
        }
    });
    
    // Preserve the selected specialty columns
    setTimeout(() => {
        selectedSpecialtyNames.forEach(specialtyName => {
            toggleSpecialtyColumn(specialtyName, true);
        });
    }, 50);
}

function setupArticleListHandlers() {
    document.querySelectorAll('.source-link').forEach(link => {
        link.addEventListener('click', function(e) {
            e.preventDefault();
            showLoader();
            
            const source = this.getAttribute('data-source');
            if (!source) return;
            
            const form = document.getElementById('filter-form');
            
            // Remove any existing source filters
            document.querySelectorAll('input[name="source_filter"]').forEach(input => {
                input.remove();
            });
            
            // Add the new source filter
            const input = document.createElement('input');
            input.type = 'hidden';
            input.name = 'source_filter';
            input.value = source;
            form.appendChild(input);
            
            form.submit();
        });
    });

    document.querySelectorAll('.cluster-badge').forEach(badge => {
        badge.addEventListener('click', function(e) {
            e.preventDefault();
            showLoader();
            
            const cluster = this.getAttribute('data-cluster');
            if (!cluster) return;

            const form = document.getElementById('filter-form');
            
            // Clear both types of cluster filters first
            document.querySelectorAll('input[name="cluster_filter"]').forEach(input => {
                input.remove();
            });
            document.querySelectorAll('input[name="specialty_cluster_filter"]').forEach(input => {
                input.remove();
            });
            
            // Reset specialty cluster state
            selectedSpecialtyClusters = {};
            
            // Reset checkboxes
            document.querySelectorAll('.specialty-checkbox').forEach(checkbox => {
                checkbox.checked = false;
            });
            
            // Add new cluster filter
            const input = document.createElement('input');
            input.type = 'hidden';
            input.name = 'cluster_filter';
            input.value = cluster;
            form.appendChild(input);
            
            const selectedClusters = document.getElementById('selected-clusters');
            if (selectedClusters) {
                selectedClusters.innerHTML = '';
                const item = document.createElement('div');
                item.className = 'selected-item';
                item.setAttribute('data-value', cluster);
                item.innerHTML = `
                    ${cluster}
                    <span class="remove" onclick="removeCluster('${cluster.replace(/'/g, "\\'")}')">✕</span>
                    <input type="hidden" name="cluster_filter" value="${cluster}">
                `;
                selectedClusters.appendChild(item);
            }
            
            form.submit();
        });
    });
}

function changePage(pageNum) {
    showLoader();
    
    // Get current URL parameters to preserve all filters including search terms
    const currentParams = new URLSearchParams(window.location.search);
    
    // Update only the page parameter
    currentParams.set('page', pageNum);
    
    // Navigate to the new URL with all parameters preserved
    window.location.href = `?${currentParams.toString()}`;
}

function sortBy(column) {
    showLoader();
    
    // Update sort inputs
    const currentSortBy = document.getElementById('sort_by').value;
    
    if (currentSortBy === column) {
        const currentSortDir = document.getElementById('sort_dir').value;
        document.getElementById('sort_dir').value = currentSortDir === 'desc' ? 'asc' : 'desc';
    } else {
        document.getElementById('sort_by').value = column;
        document.getElementById('sort_dir').value = 'desc';
    }
    
    // Make sure the column visibility state is stored before submitting
    updateColumnVisibility();
    
    // Reset to page 1 and submit
    document.getElementById('page_input').value = 1;
    document.getElementById('filter-form').submit();
}

// Make functions globally available
window.toggleSection = toggleSection;
window.clearFilters = clearFilters;
window.addSource = addSource;
window.removeSource = removeSource;
window.addCluster = addCluster;
window.removeCluster = removeCluster;
window.changePage = changePage;
window.sortBy = sortBy;
window.sortBySpecialty = sortBySpecialty;