/* hubConnect - stuff to manage connections to track hubs.  Most of this is mediated through
 * the hubStatus table in the hgcentral database.  Here there are routines to translate between
 * hub symbolic names and hub URLs,  to see if a hub is up or down or sideways (up but badly
 * formatted) etc.  Note that there is no C structure corresponding to a row in the hubStatus 
 * table by design.  We just want field-by-field access to this. */

/* Copyright (C) 2014 The Regents of the University of California 
 * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */

#include "common.h"
#include "linefile.h"
#include "hash.h"
#include "dystring.h"
#include "sqlNum.h"
#include "jksql.h"
#include "hdb.h"
#include "net.h"
#include "trackHub.h"
#include "hubConnect.h"
#include "hui.h"
#include "errCatch.h"
#include "obscure.h"
#include "hgConfig.h"
#include "grp.h"
#include "udc.h"
#include "hubPublic.h"
#include "genark.h"
#include "asmAlias.h"
#include "cheapcgi.h"

boolean isHubTrack(char *trackName)
/* Return TRUE if it's a hub track. */
{
return startsWith(hubTrackPrefix, trackName);
}

static char *hubStatusTableName = NULL;
static char *_hubPublicTableName = NULL;

static char *getHubStatusTableName()
/* return the hubStatus table name from the environment, 
 * or hg.conf, or use the default.  Cache the result */
{
if (hubStatusTableName == NULL)
    hubStatusTableName = cfgOptionEnvDefault("HGDB_HUB_STATUS_TABLE",
	    hubStatusTableConfVariable, defaultHubStatusTableName);

return hubStatusTableName;
}

char *hubPublicTableName()
/* Get the name of the table that lists public hubs.  Don't free the result. */
{
if (_hubPublicTableName == NULL)
    _hubPublicTableName = cfgOptionEnvDefault("HGDB_HUB_PUBLIC_TABLE", hubPublicTableConfVariable,
                                             defaultHubPublicTableName);
return _hubPublicTableName;
}

boolean hubConnectTableExists()
/* Return TRUE if the hubStatus table exists. */
{
struct sqlConnection *conn = hConnectCentral();
boolean exists = sqlTableExists(conn, getHubStatusTableName());
hDisconnectCentral(&conn);
return exists;
}

void hubConnectStatusFree(struct hubConnectStatus **pHub)
/* Free hubConnectStatus */
{
struct hubConnectStatus *hub = *pHub;
if (hub != NULL)
    {
    freeMem(hub->hubUrl);
    freeMem(hub->errorMessage);
    trackHubClose(&hub->trackHub);
    freez(pHub);
    }
}

void hubConnectStatusFreeList(struct hubConnectStatus **pList)
/* Free a list of dynamically allocated hubConnectStatus's */
{
struct hubConnectStatus *el, *next;

for (el = *pList; el != NULL; el = next)
    {
    next = el->next;
    hubConnectStatusFree(&el);
    }
*pList = NULL;
}

static void hubConnectRemakeTrackHubVar(struct cart *cart)
/* Remake trackHub cart variable if need be from various check box vars. */
{
if (cartVarExists(cart, hgHubConnectRemakeTrackHub))
    {
    struct slPair *hubVarList = cartVarsWithPrefix(cart, hgHubConnectHubVarPrefix);
    int prefixLength = strlen(hgHubConnectHubVarPrefix);
    struct dyString *trackHubs = dyStringNew(0);
    struct slPair *hubVar;
    boolean firstOne = TRUE;
    for (hubVar = hubVarList; hubVar != NULL; hubVar = hubVar->next)
        {
	if (cartBoolean(cart, hubVar->name))
	    {
	    if (firstOne)
		firstOne = FALSE;
	    else
		dyStringAppendC(trackHubs, ' ');
	    dyStringAppend(trackHubs, hubVar->name + prefixLength);
	    }
	}
    slPairFreeList(&hubVarList);
    cartSetString(cart, hubConnectTrackHubsVarName, trackHubs->string);
    dyStringFree(&trackHubs);
    cartRemove(cart, hgHubConnectRemakeTrackHub);
    }
}

struct slName *hubConnectHubsInCart(struct cart *cart)
/* Return list of track hub ids that are turned on. */
{
hubConnectRemakeTrackHubVar(cart);
char *trackHubString = cartOptionalString(cart, hubConnectTrackHubsVarName);
return slNameListFromString(trackHubString, ' ');
}

boolean trackHubHasDatabase(struct trackHub *hub, char *database) 
/* Return TRUE if hub has contents for database */
{
if (hub != NULL)
    {
    struct trackHubGenome *genomes = hub->genomeList;	/* List of associated genomes. */

    for(; genomes; genomes = genomes->next)
	if (sameString(genomes->name, database))
	    return TRUE;
    }
return FALSE;
}

static struct trackHub *fetchHub(struct hubConnectStatus *hubStatus, char **errorMessage)
{
struct errCatch *errCatch = errCatchNew();
struct trackHub *tHub = NULL;
boolean gotWarning = FALSE;
char *url = hubStatus->hubUrl;
*errorMessage = NULL;

char hubName[64];
safef(hubName, sizeof(hubName), "hub_%d", hubStatus->id);

if (errCatchStart(errCatch))
    tHub = trackHubOpen(url, cloneString(hubName)); // open hub
errCatchEnd(errCatch);
if (errCatch->gotError)
    {
    gotWarning = TRUE;
    *errorMessage = cloneString(errCatch->message->string);
    }
errCatchFree(&errCatch);

if (gotWarning)
    {
    return NULL;
    }

return tHub;
}

static boolean
hubTimeToCheck(struct hubConnectStatus *hub, char *notOkStatus)
/* check to see if enough time has passed to re-check a hub that
 * has an error status.  Use udcTimeout as the length of time to wait.*/
{
time_t checkTime = udcCacheTimeout();
return dateIsOlderBy(notOkStatus, "%F %T", checkTime);
}

/* Given a hub ID return associated status. Returns NULL if no such hub.  If hub
 * exists but has problems will return with errorMessage field filled in. */
struct hubConnectStatus *hubConnectStatusForId(struct sqlConnection *conn, int id)
{
struct hubConnectStatus *hub = NULL;
char query[1024];
sqlSafef(query, sizeof(query), 
    "select hubUrl,status, errorMessage,lastNotOkTime, shortLabel from %s where id=%d", getHubStatusTableName(), id);
struct sqlResult *sr = sqlGetResult(conn, query);
char **row = sqlNextRow(sr);
if (row != NULL)
    {
    AllocVar(hub);
    hub->id = id;
    hub->hubUrl = cloneString(row[0]);
    hub->status = sqlUnsigned(row[1]);
    hub->errorMessage = cloneString(row[2]);
    hub->shortLabel = cloneString(row[4]);
    if (isEmpty(row[2]) || hubTimeToCheck(hub, row[3]))
	{
	char *errorMessage = NULL;
	hub->trackHub = fetchHub( hub, &errorMessage);
        if (hub->trackHub)
            hub->trackHub->hubStatus = hub;
	hub->errorMessage = cloneString(errorMessage);
	hubUpdateStatus( hub->errorMessage, hub);
	if (!isEmpty(hub->errorMessage))
	    {
            boolean isCollection = (strstr(hub->hubUrl, "hgComposite") != NULL);
            if (isCollection)
                warn("You created a <a href=\"/cgi-bin/hgCollection\"><b>Track "
         "Collection</b></a> that has expired and been removed. Track Collections "
         "expire 48 hours after their last use. <a href=\"/cgi-bin/hgSession\"><b>"
         "Save your session</b></a> to preserve collections long-term and to allow sharing.");
            // commenting this out, but leaving it in the source because we might use it later.
            //else
                //warn("Could not connect to hub \"%s\": %s", hub->shortLabel, hub->errorMessage);
	    }
	}
    }
sqlFreeResult(&sr);
return hub;
}

struct hubConnectStatus *hubConnectStatusListFromCartAll(struct cart *cart)
/* Return list of all track hubs that are referenced by cart. */
{
struct hubConnectStatus *hubList = NULL, *hub;
struct slPair *pair, *pairList = cartVarsWithPrefix(cart, hgHubConnectHubVarPrefix);
struct sqlConnection *conn = hConnectCentral();
for (pair = pairList; pair != NULL; pair = pair->next)
    {
    // is this hub turned connected??
    if (differentString(pair->val, "1"))
        continue;

    int id = hubIdFromCartName(pair->name);
    hub = hubConnectStatusForId(conn, id);
    if (hub != NULL)
	{
        slAddHead(&hubList, hub);
	}
    }
slFreeList(&pairList);
hDisconnectCentral(&conn);
slReverse(&hubList);
return hubList;
}

struct hubConnectStatus *hubConnectStatusListFromCart(struct cart *cart)
/* Return list of track hubs that are turned on by user in cart. */
{
struct hubConnectStatus *hubList = NULL, *hub;
struct slName *name, *nameList = hubConnectHubsInCart(cart);
struct sqlConnection *conn = hConnectCentral();
for (name = nameList; name != NULL; name = name->next)
    {
    int id = sqlSigned(name->name);
    hub = hubConnectStatusForId(conn, id);
    if (hub != NULL)
	{
	if (!isEmpty(hub->errorMessage) && (strstr(hub->hubUrl, "hgComposite") != NULL))
            {
            // custom collection hub has disappeared.   Remove it from cart
            cartSetString(cart, hgHubConnectRemakeTrackHub, "on");
            char buffer[1024];
            safef(buffer, sizeof buffer, "hgHubConnect.hub.%d", id);
            cartRemove(cart, buffer);
            }
        else
            slAddHead(&hubList, hub);
	}
    }
slFreeList(&nameList);
hDisconnectCentral(&conn);
slReverse(&hubList);
return hubList;
}

int hubIdFromCartName(char *cartName)
/* Given something like "hgHubConnect.hub.123" return 123 */
{
assert(startsWith("hgHubConnect.hub.", cartName));

char *ptr1 = strchr(cartName, '.');
char *ptr2 = strchr(ptr1 + 1, '.');

return sqlUnsigned(ptr2+1);
}

unsigned hubIdFromTrackName(char *trackName)
/* Given something like "hub_123_myWig" return 123 */
{
assert(startsWith("hub_", trackName));
char *ptr1 = trackName;
ptr1 += 4;
char *ptr2 = strchr(ptr1, '_');

if (ptr2 == NULL)
    errAbort("hub track %s not in correct format\n", trackName);
char save = *ptr2;
*ptr2 = 0;
unsigned val = sqlUnsigned(ptr1);
*ptr2 = save;
return  val;
}

char *hubConnectSkipHubPrefix(char *trackName)
/* Given something like "hub_123_myWig" return myWig.  Don't free this, it's not allocated */
{
if(!startsWith("hub_", trackName))
    return trackName;

trackName += 4;
trackName = strchr(trackName, '_');
assert(trackName != NULL);
return trackName + 1;
}

struct hubConnectStatus *hubFromId(unsigned hubId)
/* Given a hub ID number, return corresponding trackHub structure. 
 * ErrAbort if there's a problem. */
{
struct sqlConnection *conn = hConnectCentral();
struct hubConnectStatus *status = hubConnectStatusForId(conn, hubId);
hDisconnectCentral(&conn);
if (status == NULL)
    errAbort("The hubId %d was not found", hubId);
if (!isEmpty(status->errorMessage))
    errAbort("%s", status->errorMessage);
return status;
}

static struct trackDb *findSuperTrack(struct trackDb *tdbList, char *trackName)
/*  discover any supertracks, and if there are some add them 
 *  to the subtrack list of the supertrack */
{
struct trackDb *tdb;
struct trackDb *p = NULL;
struct trackDb *next;

for(tdb = tdbList; tdb; tdb = next)
    {
    /* save away the next pointer becuase we may detach this node and
     * add it to its supertrack parent */
    next = tdb->next;
    if (tdb->parent != NULL && sameString(trackName, tdb->parent->track))
	{
	/* found a supertrack with the right name, add this child */
	p = tdb->parent;
	slAddHead(&p->subtracks, tdb);
	}
    }

return p;
}


void hubConnectAddDescription(char *database, struct trackDb *tdb)
/* Fetch tdb->track's html description (or nearest ancestor's non-empty description)
 * and store in tdb->html. */
{
unsigned hubId = hubIdFromTrackName(tdb->track);
struct hubConnectStatus *hub = hubFromId(hubId);
struct trackHubGenome *hubGenome = trackHubFindGenome(hub->trackHub, database);
trackHubPolishTrackNames(hub->trackHub, tdb);
trackHubAddDescription(hubGenome->trackDbFile, tdb);
}

struct trackDb *hubConnectAddHubForTrackAndFindTdb( char *database, 
    char *trackName, struct trackDb **pTdbList, struct hash *trackHash)
/* Go find hub for trackName (which will begin with hub_), and load the tracks
 * for it, appending to end of list and adding to trackHash.  Return the
 * trackDb associated with trackName. This will also fill in the html fields,
 * but just for that track and it's parents. */ 
{
unsigned hubId = hubIdFromTrackName(trackName);
struct hubConnectStatus *hub = hubFromId(hubId);
struct trackHubGenome *hubGenome = trackHubFindGenome(hub->trackHub, database);
if (hubGenome == NULL)
    errAbort("Cannot find genome %s in hub %s", database, hub->hubUrl);
boolean foundFirstGenome = FALSE;
struct trackDb *tdbList = trackHubTracksForGenome(hub->trackHub, hubGenome, NULL, &foundFirstGenome);
tdbList = trackDbLinkUpGenerations(tdbList);
tdbList = trackDbPolishAfterLinkup(tdbList, database);
//this next line causes warns to print outside of warn box on hgTrackUi
//trackDbPrioritizeContainerItems(tdbList);
trackHubPolishTrackNames(hub->trackHub, tdbList);
char *fixTrackName = cloneString(trackName);
trackHubFixName(fixTrackName);
rAddTrackListToHash(trackHash, tdbList, NULL, FALSE);
if (pTdbList != NULL)
    *pTdbList = slCat(*pTdbList, tdbList);
struct trackDb *tdb = hashFindVal(trackHash, fixTrackName);
if (tdb == NULL) 
    // superTracks aren't in the hash... look in tdbList
    tdb = findSuperTrack(tdbList, fixTrackName);

if (tdb == NULL) 
    errAbort("Can't find track %s in %s", fixTrackName, hub->trackHub->url);

/* Add html for track and parents. */
/* Note: this does NOT add the HTML for supertrack kids */
struct trackDb *parent;
for (parent = tdb; parent != NULL; parent = parent->parent)
    trackHubAddDescription(hubGenome->trackDbFile, parent);

return tdb;
}

static char *getDbList(struct trackHub *tHub, int *pCount)
/* calculate dbList for hubStatus table from trackHub */
{
struct hashEl *hel;
struct dyString *dy = dyStringNew(1024);
struct hashCookie cookie = hashFirst(tHub->genomeHash);
int dbCount = 0;
while ((hel = hashNext(&cookie)) != NULL)
    {
    dbCount++;
    dyStringPrintf(dy,"%s,", hel->name);
    }
*pCount = dbCount;

return dy->string;
}

static void insertHubUrlInStatus(char *url)
/* add a url to the hubStatus table */
{
struct sqlConnection *conn = hConnectCentral();
char query[4096];
char *statusTable = getHubStatusTableName();

sqlGetLockWithTimeout(conn, "central_hubStatus", 15);

// Try to grab a row right before we insert but after the lock.
sqlSafef(query, sizeof(query), "select id from %s where hubUrl = \"%s\"", statusTable, url);
struct sqlResult *sr = sqlGetResult(conn, query);

if (sqlNextRow(sr) == NULL)  // if we got something from this query, someone added it right before we locked it
    {
    if (sqlFieldIndex(conn, statusTable, "firstAdded") >= 0)
        sqlSafef(query, sizeof(query), "insert into %s (hubUrl,shortLabel,longLabel,dbCount,dbList,status,lastOkTime,lastNotOkTime,errorMessage,firstAdded) values (\"%s\",\"\",\"\",0,NULL,0,\"\",\"\",\"\",now())", statusTable, url);
    else
        sqlSafef(query, sizeof(query), "insert into %s (hubUrl) values (\"%s\")",
	statusTable, url);
    sqlUpdate(conn, query);
    }
sqlFreeResult(&sr);
sqlReleaseLock(conn, "central_hubStatus");
hDisconnectCentral(&conn);
}

static unsigned getHubId(char *url, char **errorMessage)
/* find id for url in hubStatus table */
{
struct sqlConnection *conn = hConnectCentral();
char query[4096];
char **row;
boolean foundOne = FALSE;
int id = 0;

char *statusTableName = getHubStatusTableName();
sqlSafef(query, sizeof(query), "select id,errorMessage from %s where hubUrl = \"%s\"", statusTableName, url);

struct sqlResult *sr = sqlGetResult(conn, query);

while ((row = sqlNextRow(sr)) != NULL)
    {
    if (foundOne)
	errAbort("more than one line in %s with hubUrl %s\n", 
	    statusTableName, url);

    foundOne = TRUE;

    char *thisId = row[0], *thisError = row[1];

    if (!isEmpty(thisError))
	*errorMessage = cloneString(thisError);

    id = sqlUnsigned(thisId);
    }
sqlFreeResult(&sr);

hDisconnectCentral(&conn);

return id;
}

static struct hubConnectStatus *getAndSetHubStatus( struct cart *cart, char *url, 
    boolean set)
/* make sure url is in hubStatus table, fetch the hub to get latest
 * labels and db information.
 * Set the cart variable to turn the hub on if set == TRUE.  
 */
{
char *errorMessage = NULL;
unsigned id;

/* first see if url is in hubStatus table */
if ((id = getHubId(url, &errorMessage)) == 0)
    {
    /* the url is not in the hubStatus table, add it */
    insertHubUrlInStatus(url);
    if ((id = getHubId(url, &errorMessage)) == 0)
	{
	errAbort("opened hub, but could not get it out of the hubStatus table");
	}
    }

/* allocate a hub */
struct hubConnectStatus *hub = NULL;

AllocVar(hub);
hub->id = id;
hub->hubUrl = cloneString(url);

/* new fetch the contents of the hub to fill in the status table */
struct trackHub *tHub = fetchHub( hub, &errorMessage);
if (tHub != NULL)
    hub->trackHub = tHub;
if (errorMessage != NULL)
    hub->errorMessage = cloneString(errorMessage);

/* update the status table with the lastest label and database information */
hubUpdateStatus( errorMessage, hub);

/* if we're turning on the hub, set the cart variable */
if (set)
    {
    char hubName[32];
    safef(hubName, sizeof(hubName), "%s%u", hgHubConnectHubVarPrefix, id);
    cartSetString(cart, hubName, "1");
    }

return hub;
}

unsigned hubFindOrAddUrlInStatusTable(struct cart *cart, char *url, char **errorMessage)
/* find this url in the status table, and return its id and errorMessage (if an errorMessage exists) */
{
int id = 0;

*errorMessage = NULL;

if ((id = getHubId(url, errorMessage)) > 0)
    return id;

getAndSetHubStatus( cart, url, FALSE);

if ((id = getHubId(url, errorMessage)) == 0)
    errAbort("inserted new hubUrl %s, but cannot find it", url);

return id;
}

// global to hold hubUrl we added if any
static struct hubConnectStatus *gNewHub = NULL;

struct hubConnectStatus  *hubConnectNewHub()
/* return the hub for the hubUrl we added (if any) */
{
return gNewHub;
}

static void disconnectHubsSamePrefix(struct cart *cart, char *url)
/* disconnect all the hubs with the same prefix */
{
char *prefix = cloneString(url);
char *ptr = strrchr(prefix, '/');
if (ptr == NULL)
    return;
*ptr = 0;

char query[2048];
sqlSafef(query, sizeof(query), "select id from %s where hubUrl like  \"%s%%\"", getHubStatusTableName(), prefix);

struct sqlConnection *conn = hConnectCentral();
struct sqlResult *sr = sqlGetResult(conn, query);
char **row;
while ((row = sqlNextRow(sr)) != NULL)
    {
    int id = sqlUnsigned(row[0]);
    char buffer[512];

    safef(buffer, sizeof(buffer), "hgHubConnect.hub.%d", id);
    cartRemove(cart, buffer);
    }
sqlFreeResult(&sr);

hDisconnectCentral(&conn);
}

static char  *checkForNew( struct cart *cart)
/* see if the user just typed in a new hub url, or we have one or more hubUrl 
 * on the command line.  Return the new database if there is one. */
{
char *newDatabase = NULL;
boolean doClear = FALSE;
char *assemblyDb = cartOptionalString(cart, hgHubGenome);
char *wantFirstDb = cartOptionalString(cart, hgHubDoFirstDb);

struct slName *urls = cartOptionalSlNameList(cart, hgHubDataClearText);
if (urls)
    doClear = TRUE;
else
    urls  = cartOptionalSlNameList(cart, hgHubDataText);

if (urls == NULL)
    return NULL;

for(; urls; urls = urls->next)
    {
    char *url = cloneString(urls->name);
    if (doClear)
        disconnectHubsSamePrefix(cart, url);

    trimSpaces(url);

    // go and grab the hub and set the cart variables to connect it
    struct hubConnectStatus *hub;
    gNewHub = hub = getAndSetHubStatus( cart, url, TRUE);

    if (newDatabase == NULL)  // if we haven't picked a new database yet
        {
        if ((wantFirstDb != NULL) && (hub->trackHub != NULL)) // choose the first db
            newDatabase = hub->trackHub->defaultDb;
        else if (assemblyDb != NULL)
            {
            // Check to see if the user specified an assembly within
            // an assembly hub.
            struct trackHub *trackHub = hub->trackHub;
            if (trackHub != NULL)
                {
                struct trackHubGenome *genomeList = trackHub->genomeList;

                for(; genomeList; genomeList = genomeList->next)
                    {
                    if (sameString(assemblyDb, hubConnectSkipHubPrefix(genomeList->name)))
                        {
                        if (hub->errorMessage)
                            errAbort("Hub error: %s", hub->errorMessage);
                        newDatabase = genomeList->name;
                        break;
                        }
                    }
                }
            }
        }
    }

cartRemove(cart, hgHubDataClearText);
cartRemove(cart, hgHubDataText);
cartRemove(cart, hgHubDoFirstDb);
cartRemove(cart, hgHubGenome);
return newDatabase;
}

unsigned hubResetError(char *url)
/* clear the error for this url in the hubStatus table,return the id */
{
struct sqlConnection *conn = hConnectCentral();
char query[512];

sqlSafef(query, sizeof(query), "select id from %s where hubUrl = \"%s\"", getHubStatusTableName(), url);
unsigned id = sqlQuickNum(conn, query);

if (id == 0)
    errAbort("could not find url %s in status table (%s)\n", 
	url, getHubStatusTableName());

sqlSafef(query, sizeof(query), "update %s set errorMessage=\"\" where hubUrl = \"%s\"", getHubStatusTableName(), url);

sqlUpdate(conn, query);
hDisconnectCentral(&conn);

return id;
}

unsigned hubClearStatus(char *url)
/* drop the information about this url from the hubStatus table */
{
struct sqlConnection *conn = hConnectCentral();
char query[512];

sqlSafef(query, sizeof(query), "select id from %s where hubUrl = \"%s\"", getHubStatusTableName(), url);
unsigned id = sqlQuickNum(conn, query);

if (id == 0)
    errAbort("could not find url %s in status table (%s)\n", 
	url, getHubStatusTableName());

sqlSafef(query, sizeof(query), "delete from %s where hubUrl = \"%s\"", getHubStatusTableName(), url);

sqlUpdate(conn, query);
hDisconnectCentral(&conn);

return id;
}

void hubDisconnect(struct cart *cart, char *url)
/* drop the information about this url from the hubStatus table, and 
 * the cart variable the references this hub */
{
/* clear the hubStatus table */
unsigned id = hubClearStatus(url);

/* remove the cart variable */
char buffer[1024];
safef(buffer, sizeof buffer, "hgHubConnect.hub.%d", id);
cartRemove(cart, buffer);
}

void hubUpdateStatus(char *errorMessage, struct hubConnectStatus *hub)
/* set the error message in the hubStatus table */
{
struct sqlConnection *conn = hConnectCentral();
char query[64 * 1024];
struct trackHub *tHub = NULL;

if (hub != NULL)
    tHub = hub->trackHub;

if (errorMessage != NULL)
    {
    // make sure there is no newline at the end.  This should be unneccesary
    // but there are many, many places where newlines are added in calls
    // to warn and errAbort
    char buffer[64 * 1024];
    safecpy(buffer, sizeof buffer, errorMessage);
    while (lastChar(buffer) == '\n')
	buffer[strlen(buffer) - 1] = '\0';
    sqlSafef(query, sizeof(query),
	"update %s set errorMessage=\"%s\", lastNotOkTime=now() where id=%d",
	getHubStatusTableName(), buffer, hub->id);
    sqlUpdate(conn, query);
    }
else if (tHub != NULL)
    {
    int dbCount = 0;
    char *dbList = getDbList(tHub, &dbCount);
    // users may include quotes in their hub names requiring escaping
    sqlSafef(query, sizeof(query),
	"update %s set shortLabel=\"%s\",longLabel=\"%s\",dbCount=\"%d\",dbList=\"%s\",errorMessage=\"\",lastOkTime=now() where id=%d",
	getHubStatusTableName(), tHub->shortLabel, tHub->longLabel, 
	dbCount, dbList,
	hub->id);
    sqlUpdate(conn, query);
    }
hDisconnectCentral(&conn);
}

struct trackDb *hubAddTracks(struct hubConnectStatus *hub, char *database, boolean *foundFirstGenome, struct hash *trackDbNameHash)
/* Load up stuff from data hub and append to list. The hubUrl points to
 * a trackDb.ra format file. Only the first example of a genome gets to 
 * populate groups, the others get a group for the trackHub.  A particular 
 * trackDb is only read once even if referenced from more than one hub.  */
{
struct trackDb *tdbList = NULL;
struct trackHub *trackHub = hub->trackHub;

if (trackHub != NULL)
    {
    struct trackHubGenome *hubGenome = trackHubFindGenome(trackHub, database);
    if ((hubGenome == NULL) || hashLookup(trackDbNameHash,  hubGenome->trackDbFile))
        hubGenome = NULL; // we already saw this trackDb, so ignore this stanza
    else
        hashStore(trackDbNameHash,  hubGenome->trackDbFile);

    if (hubGenome != NULL)
	{
        boolean doCache = trackDbCacheOn();

        if (doCache)
            {
            // we have to open the trackDb file to get the udc cache to check for an update
            struct udcFile *checkCache = udcFileMayOpen(hubGenome->trackDbFile, NULL);
            if (checkCache != NULL)
                {
                time_t time = udcUpdateTime(checkCache);
                udcFileClose(&checkCache);

                struct trackDb *cacheTdb = trackDbHubCache(hubGenome->trackDbFile, time);

                if (cacheTdb != NULL)
                    return cacheTdb;
                }

            memCheckPoint(); // we want to know how much memory is used to build the tdbList
            }

        struct dyString *incFiles = dyStringNew(4096);
        tdbList = trackHubTracksForGenome(trackHub, hubGenome, incFiles, foundFirstGenome);
        tdbList = trackDbLinkUpGenerations(tdbList);
        tdbList = trackDbPolishAfterLinkup(tdbList, database);
        trackDbPrioritizeContainerItems(tdbList);
        trackHubPolishTrackNames(trackHub, tdbList);

        if (doCache)
            trackDbHubCloneTdbListToSharedMem(hubGenome->trackDbFile, tdbList, memCheckPoint(), incFiles->string);
	}
    }
return tdbList;
}

static struct grp *grpFromHub(struct hubConnectStatus *hub)
/* Make up a grp structur from hub */
{
struct grp *grp;
AllocVar(grp);
char name[16];
safef(name, sizeof(name), "hub_%d", hub->id);
grp->name = cloneString(name);
grp->label = cloneString(hub->shortLabel);
return grp;
}

struct trackDb *hubCollectTracks( char *database,  struct grp **pGroupList)
/* Generate trackDb structures for all the tracks in attached hubs.  
 * Make grp structures for each hub. Returned group list is reversed. */
{
// return the cached copy if it exists
static struct trackDb *hubTrackDbs;
static struct grp *hubGroups;

if (hubTrackDbs != NULL)
    {
    if (pGroupList != NULL)
	*pGroupList = hubGroups;
    return hubTrackDbs;
    }

struct hubConnectStatus *hub, *hubList =  hubConnectGetHubs();
struct trackDb *tdbList = NULL;
boolean foundFirstGenome = FALSE;
struct hash *trackDbNameHash = newHash(5);
for (hub = hubList; hub != NULL; hub = hub->next)
    {
    if (isEmpty(hub->errorMessage))
	{
        /* error catching in so it won't just abort  */
        struct errCatch *errCatch = errCatchNew();
        if (errCatchStart(errCatch))
	    {
	    struct trackDb *thisList = hubAddTracks(hub, database, &foundFirstGenome, trackDbNameHash);
	    tdbList = slCat(tdbList, thisList);
	    }
        errCatchEnd(errCatch);
        if (errCatch->gotError)
	    {
	    warn("%s", errCatch->message->string);
	    hubUpdateStatus( errCatch->message->string, hub);
	    }
	else
	    {
            struct grp *grp = grpFromHub(hub);
            slAddHead(&hubGroups, grp);
	    hubUpdateStatus(NULL, hub);
	    }

        errCatchFree(&errCatch);
	}
    else
        {
        /* create an empty group to hold the error message. */
        struct grp *grp = grpFromHub(hub);
        grp->errMessage = hub->errorMessage;
        slAddHead(&hubGroups, grp);
        }
    }

hubTrackDbs = tdbList;
if (pGroupList != NULL)
    *pGroupList = hubGroups;
return tdbList;
}

static struct hubConnectStatus *globalHubList;

struct hubConnectStatus *hubConnectGetHubs()
/* return the static global to the track data hubs */
{
return globalHubList;
}

struct trackHub *hubConnectGetHub(char *hubUrl)
/* Return the connected hub for hubUrl, or NULL if not found.  Do not free result. */
{
struct hubConnectStatus *status;
for (status = globalHubList;  status != NULL;  status = status->next)
    {
    if (sameString(status->hubUrl, hubUrl))
        return status->trackHub;
    }
return NULL;
}

struct trackHub *hubConnectGetHubForDb(char *db)
/* Return the connected hub for db, or NULL if not found.  Do not free result. */
{
unsigned hubId = hubIdFromTrackName(db);
struct hubConnectStatus *status;
for (status = globalHubList;  status != NULL;  status = status->next)
    {
    if (status->id == hubId)
        return status->trackHub;
    }
return NULL;
}

static unsigned lookForUndecoratedDb(char *name)
// Look for this undecorated in the attached assembly hubs
{
struct trackHubGenome *genome = trackHubGetGenomeUndecorated(name);

if (genome == NULL)
    return FALSE;

struct trackHub *trackHub = genome->trackHub;

if ((trackHub != NULL) && (trackHub->hubStatus != NULL))
    return trackHub->hubStatus->id;
return 0;
}

static boolean lookForLonelyHubs(struct cart *cart, struct hubConnectStatus  *hubList, char **newDatabase, char *genarkPrefix)
// We go through the hubs and see if any of them reference an assembly
// that is NOT currently loaded, but we know a URL to load it.
{
struct sqlConnection *conn = hConnectCentral();
if (!sqlTableExists(conn, genarkTableName()))
    return FALSE;

boolean added = FALSE;

struct hubConnectStatus  *hub;
for(hub = hubList; hub; hub = hub->next)
    {
    struct trackHub *tHub = hub->trackHub;
    if (tHub == NULL)
        continue;

    struct trackHubGenome *genomeList = tHub->genomeList, *genome;

    for(genome = genomeList; genome; genome = genome->next)
        {
        char *name = genome->name;

        name = asmAliasFind(name);
        if (!hDbIsActive(name) )
            {
            char buffer[4096];
            unsigned newId = 0;

            // look with undecorated name for an attached assembly hub
            if (!(newId = lookForUndecoratedDb(name)))
                {
                // see if genark has this assembly
                char query[4096];
                sqlSafef(query, sizeof query, "select hubUrl from %s where gcAccession='%s'", genarkTableName(), name);
                if (sqlQuickQuery(conn, query, buffer, sizeof buffer))
                    {
                    char url[4096];
                    safef(url, sizeof url, "%s/%s", genarkPrefix, buffer);

                    struct hubConnectStatus *status = getAndSetHubStatus( cart, url, TRUE);

                    if (status)
                        {
                        newId = status->id;
                        added = TRUE;
                        }
                    }
                }

            // if we found an id, change some names to use it as a decoration
            if (newId)
                {
                safef(buffer, sizeof buffer, "hub_%d_%s", newId, name);

                genome->name = cloneString(buffer);
    
                // if our new database is an undecorated db, decorate it
                if (*newDatabase && sameString(*newDatabase, name))
                    *newDatabase = cloneString(buffer);
                }

            }
        }
    }

hDisconnectCentral(&conn);
return added;
}

static char *getCuratedHubPrefix()
/* figure out what sandbox we're in. */
{
char *curatedHubPrefix = cfgOption("curatedHubPrefix");
if (isEmpty(curatedHubPrefix))
    curatedHubPrefix = "public";

return curatedHubPrefix;
}


boolean hubConnectGetCuratedUrl(char *db, char **hubUrl)
/* Check to see if this db is a curated hub and if so return its hubUrl */
{
struct sqlConnection *conn = hConnectCentral();
char query[4096];
sqlSafef(query, sizeof query, "SELECT nibPath from %s where name = '%s' AND nibPath like '%s%%'",
          dbDbTable(), db, hubCuratedPrefix);

char *dir = sqlQuickString(conn, query);
boolean ret = !isEmpty(dir);
hDisconnectCentral(&conn);

if (hubUrl != NULL) // if user passed in hubUrl, calculate what it should be
    {
    *hubUrl = NULL;
    if (!isEmpty(dir))   // this is a curated hub
        {
        char *path = dir + sizeof(hubCuratedPrefix) - 1;
        char url[4096];
        safef(url, sizeof url, "%s/%s/hub.txt", path, getCuratedHubPrefix());
        *hubUrl = cloneString(url);
        }
    }

return ret;
}

boolean hubConnectIsCurated(char *db)
/* Look in the dbDb table to see if this hub is curated. */
{
return hubConnectGetCuratedUrl(db, NULL);
}

static int lookForCuratedHubs(struct cart *cart, char *db,  char *curatedHubPrefix)
/* Check to see if db is a curated hub which will require the hub to be attached. 
 * The variable curatedHubPrefix has the release to use (alpha, beta, public, or a user name ) */
{
struct sqlConnection *conn = hConnectCentral();
char query[4096];
sqlSafef(query, sizeof query, "SELECT nibPath from %s where name = '%s' AND nibPath like '%s%%'",
          dbDbTable(), db, hubCuratedPrefix);

char *dir = cloneString(sqlQuickString(conn, query));
hDisconnectCentral(&conn);

if (!isEmpty(dir))
    {
    char *path = &dir[sizeof hubCuratedPrefix - 1];
    char url[4096];
    safef(url, sizeof url, "%s/%s/hub.txt", path, curatedHubPrefix);

    struct hubConnectStatus *status = getAndSetHubStatus( cart, url, TRUE);

    if (status && isEmpty(status->errorMessage))
        {
        char buffer[4096];
        safef(buffer, sizeof buffer, "hub_%d_%s", status->id, db);

        cartSetString(cart, "db", buffer);
        if (cgiOptionalString("db"))
            {
            /* user specified db on URL, we need to decorate and put it back. */
            cgiVarSet("db",  cloneString(buffer));
            }

        return status->id;
        }
    else
        {
        if (!isEmpty(status->errorMessage))
            errAbort("Hub error: url %s: error  %s.", url, status->errorMessage);
        else
            errAbort("Cannot open hub %s.", url);
        }

    }
return 0;
}

static void portHubStatus(struct cart *cart)
/* When a session has been saved on a different host it may have cart variables that reference hubStatus id's
 * that are different than what the local machine has.  Look for these cases using the "assumesHub" cart variable
 * that maps id numbers to URL's. */
{
char *assumesHubStr = cartOptionalString(cart, "assumesHub");
if (assumesHubStr == NULL)
    return;

struct slPair *pairs = slPairListFromString(assumesHubStr, FALSE);
for(; pairs; pairs = pairs->next)
    {
    char *url = (char *)pairs->val;
    unsigned sessionHubId = atoi(pairs->name);
    char *errMessage;
    unsigned localHubId = hubFindOrAddUrlInStatusTable(cart, url, &errMessage);

    if (localHubId != sessionHubId)  // the hubStatus id on the local machine is different than in the session
        {
        // first look for visibility and settings for tracks on this hub
        char wildCard[4096];

        safef(wildCard, sizeof wildCard, "hub_%d_*", sessionHubId);
        struct slPair *cartVars = cartVarsLike(cart, wildCard);

        for(; cartVars; cartVars = cartVars->next)
            {
            char *rest = trackHubSkipHubName(cartVars->name);
            char newVarName[4096];
            
            // add the new visibility/setting
            safef(newVarName, sizeof newVarName, "hub_%d_%s", localHubId, rest);
            cartSetString(cart, newVarName, cartVars->val);

            // remove the old visibility/setting
            cartRemove(cart, cartVars->name);
            }

        // turn on this remapped hub
        char hubName[4096];
        cartSetString(cart, hgHubConnectRemakeTrackHub, "on");
        safef(hubName, sizeof(hubName), "%s%u", hgHubConnectHubVarPrefix, localHubId);
        cartSetString(cart, hubName, "1");

        // remove the old hub connection
        safef(hubName, sizeof(hubName), "%s%u", hgHubConnectHubVarPrefix, sessionHubId);
        cartRemove(cart, hubName);
        }
    }

cartRemove(cart, "assumesHub");
}

char *hubConnectLoadHubs(struct cart *cart)
/* load the track data hubs.  Set a static global to remember them */
{
char *dbSpec = asmAliasFind(cartOptionalString(cart, "db"));
char *curatedHubPrefix = getCuratedHubPrefix();
if (dbSpec != NULL)
    lookForCuratedHubs(cart, trackHubSkipHubName(dbSpec), curatedHubPrefix);

char *newDatabase = checkForNew( cart);
newDatabase = asmAliasFind(newDatabase);
cartSetString(cart, hgHubConnectRemakeTrackHub, "on");

portHubStatus(cart);

struct hubConnectStatus  *hubList =  hubConnectStatusListFromCart(cart);

char *genarkPrefix = cfgOption("genarkHubPrefix");
if (genarkPrefix && lookForLonelyHubs(cart, hubList, &newDatabase, genarkPrefix))
    hubList = hubConnectStatusListFromCart(cart);

globalHubList = hubList;

return newDatabase;
}

char *hubNameFromUrl(char *hubUrl)
/* Given the URL for a hub, return its hub_# name. */
{
char query[PATH_LEN*4];
sqlSafef(query, sizeof(query), "select concat('hub_', id) from %s where hubUrl = '%s'",
         getHubStatusTableName(), hubUrl);
struct sqlConnection *conn = hConnectCentral();
char *name = sqlQuickString(conn, query);
hDisconnectCentral(&conn);
return name;
}

void addPublicHubsToHubStatus(struct cart *cart, struct sqlConnection *conn, char *publicTable, char  *statusTable)
/* Add urls in the hubPublic table to the hubStatus table if they aren't there already */
{
char query[1024];
sqlSafef(query, sizeof(query),
        "select %s.hubUrl from %s left join %s on %s.hubUrl = %s.hubUrl where %s.hubUrl is NULL",
        publicTable, publicTable, statusTable, publicTable, statusTable, statusTable);
struct sqlResult *sr = sqlGetResult(conn, query);
char **row;
while ((row = sqlNextRow(sr)) != NULL)
    {
    char *errorMessage = NULL;
    char *url = row[0];

    // add this url to the hubStatus table
    hubFindOrAddUrlInStatusTable(cart, url, &errorMessage);
    }
}

struct hash *buildPublicLookupHash(struct sqlConnection *conn, char *publicTable, char *statusTable,
        struct hash **pHash)
/* Return a hash linking hub URLs to struct hubEntries.  Also make pHash point to a hash that just stores
 * the names of the public hubs (for use later when determining if hubs were added by the user) */
{
struct hash *hubLookup = newHash(5);
struct hash *publicLookup = newHash(5);
char query[512];
bool hasDescription = sqlColumnExists(conn, publicTable, "descriptionUrl");
if (hasDescription)
    sqlSafef(query, sizeof(query), "select p.hubUrl,p.shortLabel,p.longLabel,p.dbList,"
            "s.errorMessage,s.id,p.descriptionUrl from %s p,%s s where p.hubUrl = s.hubUrl",
	    publicTable, statusTable);
else
    sqlSafef(query, sizeof(query), "select p.hubUrl,p.shortLabel,p.longLabel,p.dbList,"
            "s.errorMessage,s.id from %s p,%s s where p.hubUrl = s.hubUrl",
	    publicTable, statusTable);

struct sqlResult *sr = sqlGetResult(conn, query);
char **row;
while ((row = sqlNextRow(sr)) != NULL)
    {
    struct hubEntry *hubInfo = hubEntryTextLoad(row, hasDescription);
    hubInfo->tableHasDescriptionField = hasDescription;
    hashAddUnique(hubLookup, hubInfo->hubUrl, hubInfo);
    hashStore(publicLookup, hubInfo->hubUrl);
    }
sqlFreeResult(&sr);
*pHash = publicLookup;
return hubLookup;
}