/* Import - put up import pages and sub-pages. */ /* Copyright (C) 2013 The Regents of the University of California * See kent/LICENSE or http://genome.ucsc.edu/license/ for licensing information. */ #include "common.h" #include "linefile.h" #include "hash.h" #include "ra.h" #include "portable.h" #include "cheapcgi.h" #include "localmem.h" #include "cart.h" #include "web.h" #include "chromInfo.h" #include "chromGraph.h" #include "chromGraphFactory.h" #include "errCatch.h" #include "hPrint.h" #include "customTrack.h" #include "trackDb.h" #include "trashDir.h" #include "hgGenome.h" #include "genoLay.h" #include "jsHelper.h" #include "grp.h" #include "hdb.h" #include "joiner.h" #include "hgMaf.h" #include "wiggle.h" /* from hgTables.c */ struct grp *fullGroupList; /* List of all groups. */ struct grp *curGroup; /* Currently selected group. */ struct trackDb *fullTrackList; /* List of all tracks in database. */ struct trackDb *curTrack; /* Currently selected track. */ char *curTable; /* Currently selected table. */ struct joiner *allJoiner; /* Info on how to join tables. */ char *getScriptName() /* returns script name from environment or hardcoded for command line */ { char *script = cgiScriptName(); if (script != NULL) return script; else return "hgGenome"; } struct genoLayChrom *getChroms() /* Get a chrom list. */ { struct sqlConnection *conn = hAllocConn(database); struct genoLayChrom *chromList = genoLayDbChromsExt(conn, FALSE, FALSE); hFreeConn(&conn); return chromList; } struct sqlResult *chromQuery(struct sqlConnection *conn, char *table, char *fields, char *chrom, boolean isPositional, char *extraWhere) /* Construct and execute query for table on chrom. Returns NULL if * table doesn't exist (e.g. missing split table for chrom). */ { struct sqlResult *sr; if (isPositional) { /* Check for missing split tables before querying: */ char *db = sqlGetDatabase(conn); struct hTableInfo *hti = hFindTableInfo(db, chrom, table); if (hti == NULL) return NULL; else if (hti->isSplit) { char fullTableName[256]; safef(fullTableName, sizeof(fullTableName), "%s_%s", chrom, table); if (!sqlTableExists(conn, fullTableName)) return NULL; } sr = hExtendedChromQuery(conn, table, chrom, extraWhere, FALSE, fields, NULL); } else { struct dyString *query = dyStringNew(0); sqlDyStringPrintf(query, "select %s from %s", fields, table); if (extraWhere) { sqlDyStringPrintf(query, " where %-s", extraWhere); } sr = sqlGetResult(conn, query->string); dyStringFree(&query); } return sr; } struct hTableInfo *maybeGetHti(char *db, char *table) /* Return primary table info. */ { struct hTableInfo *hti = NULL; if (isCustomTrack(table)) { struct customTrack *ct = lookupCt(table); hti = ctToHti(ct); } else { char *track; if (startsWith("chrN_", table)) track = table + strlen("chrN_"); else track = table; hti = hFindTableInfo(db, NULL, track); } return(hti); } struct hTableInfo *getHti(char *db, char *table) /* Return primary table info. */ { struct hTableInfo *hti = maybeGetHti(db, table); if (hti == NULL) { errAbort("Could not find table info for table %s in db %s", table, db); } return(hti); } struct grp *makeGroupList(struct trackDb *trackList, boolean allTablesOk) /* Get list of groups that actually have something in them. */ { struct grp *groupsAll, *groupList = NULL, *group; struct hash *groupsInTrackList = newHash(0); struct hash *groupsInDatabase = newHash(0); struct trackDb *track; /* Stream throught track list building up hash of active groups. */ for (track = trackList; track != NULL; track = track->next) { if (!hashLookup(groupsInTrackList,track->grp)) hashAdd(groupsInTrackList, track->grp, NULL); } /* Scan through group table, putting in ones where we have data. */ groupsAll = hLoadGrps(database); for (group = slPopHead(&groupsAll); group != NULL; group = slPopHead(&groupsAll)) { if (hashLookup(groupsInTrackList, group->name)) { slAddTail(&groupList, group); hashAdd(groupsInDatabase, group->name, group); } else grpFree(&group); } /* Do some error checking for tracks with group names that are * not in database. Just warn about them. */ for (track = trackList; track != NULL; track = track->next) { if (!hashLookup(groupsInDatabase, track->grp)) warn("Track %s has group %s, which isn't in grp table", track->track, track->grp); } /* Create dummy group for all tracks. */ AllocVar(group); group->name = cloneString("allTracks"); group->label = cloneString("All Tracks"); slAddTail(&groupList, group); /* Create another dummy group for all tables. */ if (allTablesOk) { AllocVar(group); group->name = cloneString("allTables"); group->label = cloneString("All Tables"); slAddTail(&groupList, group); } hashFree(&groupsInTrackList); hashFree(&groupsInDatabase); return groupList; } static struct grp *findGroup(struct grp *groupList, char *name) /* Return named group in list, or NULL if not found. */ { struct grp *group; for (group = groupList; group != NULL; group = group->next) if (sameString(name, group->name)) return group; return NULL; } struct grp *findSelectedGroup(struct grp *groupList, char *cgiVar) /* Find user-selected group if possible. If not then * go to various levels of defaults. */ { char *defaultGroup = "genes"; char *name = cartUsualString(cart, cgiVar, defaultGroup); struct grp *group = findGroup(groupList, name); if (group == NULL) group = findGroup(groupList, defaultGroup); if (group == NULL) group = groupList; return group; } struct trackDb *findTrackInGroup(char *name, struct trackDb *trackList, struct grp *group) /* Find named track that is in group (NULL for any group). * Return NULL if can't find it. */ { struct trackDb *track; if (group != NULL && sameString(group->name, "all")) group = NULL; for (track = trackList; track != NULL; track = track->next) { if (track->table != NULL && sameString(name, track->table) && (group == NULL || sameString(group->name, track->grp))) return track; } return NULL; } struct trackDb *findTrack(char *name, struct trackDb *trackList) /* Find track, or return NULL if can't find it. */ { return findTrackInGroup(name, trackList, NULL); } struct trackDb *findSelectedTrack(struct trackDb *trackList, struct grp *group, char *varName) /* Find selected track - from CGI variable if possible, else * via various defaults. */ { char *name = cartOptionalString(cart, varName); struct trackDb *track = NULL; if (name != NULL) { track = findTrackInGroup(name, trackList, group); } if (track == NULL) { if (group == NULL || sameString(group->name, "all")) track = trackList; else { for (track = trackList; track != NULL; track = track->next) if (sameString(track->grp, group->name)) break; if (track == NULL) internalErr(); } } return track; } struct trackDb *getFullTrackList() /* Get all tracks including custom tracks if any. */ { struct trackDb *list = hTrackDb(database); struct customTrack *ctList, *ct; /* Create dummy group for custom tracks if any */ ctList = getCustomTracks(); for (ct = ctList; ct != NULL; ct = ct->next) { if (!isChromGraph(ct->tdb)) /* filter out chromGraph custom tracks */ slAddHead(&list, ct->tdb); } return list; } static void addTablesAccordingToTrackType(struct slName **pList, struct hash *uniqHash, struct trackDb *track) /* Parse out track->type and if necessary add some tables from it. */ { struct slName *name; char *trackDupe = cloneString(track->type); if (trackDupe != NULL && trackDupe[0] != 0) { char *s = trackDupe; char *type = nextWord(&s); if (sameString(type, "wigMaf")) { static char *wigMafAssociates[] = {"frames", "summary"}; int i; for (i=0; inext) { name = slNameNew(wig->table); slAddHead(pList, name); hashAdd(uniqHash, wig->table, NULL); } } if (tdbIsContainer(track)) // was tdbIsComposite(track)) but missed Jim's new { // regulation track which isn't quite a composite struct slName *subList = NULL; struct slRef *tdbRefList = trackDbListGetRefsToDescendantLeaves(track->subtracks); slSort(&tdbRefList, trackDbRefCmp); struct slRef *tdbRef; for (tdbRef = tdbRefList; tdbRef != NULL; tdbRef = tdbRef->next) { struct trackDb *subTdb = tdbRef->val; if (!tdbIsDownloadsOnly(subTdb) && subTdb->table != NULL) { name = slNameNew(subTdb->table); slAddTail(&subList, name); hashAdd(uniqHash, subTdb->table, NULL); } } pList = slCat(pList, subList); } } freez(&trackDupe); } struct slName *tablesForTrack(struct trackDb *track, boolean useJoiner) /* Return list of all tables associated with track. */ { struct hash *uniqHash = newHash(8); struct slName *name, *nameList = NULL; char *trackTable = track->table ? track->table : track->track; hashAdd(uniqHash, trackTable, NULL); if (useJoiner) { struct joinerPair *jpList, *jp; jpList = joinerRelate(allJoiner, database, trackTable, NULL); for (jp = jpList; jp != NULL; jp = jp->next) { struct joinerDtf *dtf = jp->b; char buf[256]; char *s; if (sameString(dtf->database, database)) s = dtf->table; else { safef(buf, sizeof(buf), "%s.%s", dtf->database, dtf->table); s = buf; } if (!hashLookup(uniqHash, s)) { hashAdd(uniqHash, s, NULL); name = slNameNew(s); slAddHead(&nameList, name); } } slNameSort(&nameList); } name = slNameNew(trackTable); if (!tdbIsComposite(track)) /* suppress for composite tracks -- only the subtracks have tables */ slAddHead(&nameList, name); addTablesAccordingToTrackType(&nameList, uniqHash, track); hashFree(&uniqHash); return nameList; } static char *findSelectedTable(struct trackDb *track, char *var) /* Find selected table. Default to main track table if none * found. */ { if (track == NULL) return cartString(cart, var); else if (isCustomTrack(track->table)) return track->table; else { struct slName *tableList = tablesForTrack(track, TRUE); char *table = cartUsualString(cart, var, tableList->name); if (slNameInList(tableList, table)) return table; return tableList->name; } } void initGroupsTracksTables() /* Get list of groups that actually have something in them. */ { fullTrackList = getFullTrackList(); curTrack = findSelectedTrack(fullTrackList, NULL, hggTrack); fullGroupList = makeGroupList(fullTrackList, TRUE); curGroup = findSelectedGroup(fullGroupList, hggGroup); if (sameString(curGroup->name, "allTables")) curTrack = NULL; curTable = findSelectedTable(curTrack, hggTable); if (curTrack == NULL) { struct trackDb *tdb = hTrackDbForTrack(database, curTable); struct trackDb *cTdb = hCompositeTrackDbForSubtrack(database, tdb); if (cTdb) curTrack = cTdb; else curTrack = tdb; } } /* --------------------- */ void nbSpaces(int count) /* Print some non-breaking spaces. */ { int i; for (i=0; igroup:\n"); hPrintf("\n"); return selGroup; } static void addIfExists(struct hash *hash, struct slName **pList, char *name) /* Add name to tail of list if it exists in hash. */ { if (hashLookup(hash, name)) slNameAddTail(pList, name); } struct slName *getDbListForGenome() /* Get list of selectable databases. */ { struct hash *hash = sqlHashOfDatabases(); struct slName *dbList = NULL; addIfExists(hash, &dbList, database); /* currently filtering these out for hgGenome addIfExists(hash, &dbList, "swissProt"); addIfExists(hash, &dbList, "proteins"); addIfExists(hash, &dbList, "uniProt"); addIfExists(hash, &dbList, "proteome"); addIfExists(hash, &dbList, "go"); addIfExists(hash, &dbList, "hgFixed"); addIfExists(hash, &dbList, "visiGene"); addIfExists(hash, &dbList, "ultra"); */ return dbList; } char *findSelDb() /* Find user selected database (as opposed to genome database). */ { struct slName *dbList = getDbListForGenome(); char *selDb = cartUsualString(cart, hggTrack, NULL); if (!slNameInList(dbList, selDb)) selDb = cloneString(dbList->name); slFreeList(&dbList); return selDb; } struct trackDb *showTrackField(struct grp *selGroup, char *trackVar, char *trackScript) /* Show track control. Returns selected track. */ { struct trackDb *track, *selTrack = NULL; if (trackScript == NULL) trackScript = ""; if (sameString(selGroup->name, "allTables")) { char *selDb = findSelDb(); struct slName *dbList = getDbListForGenome(), *db; hPrintf("database:\n"); hPrintf("\n"); } else { boolean allTracks = sameString(selGroup->name, "allTracks"); hPrintf("track:\n"); hPrintf("\n"); } hPrintf("\n"); return selTrack; } struct trackDb *findCompositeTdb(struct trackDb *track, char *table) /* find the tdb for the table, if it is custom or composite or ordinary */ { struct trackDb *tdb = track; if (isCustomTrack(table)) { struct customTrack *ct = lookupCt(table); tdb = ct->tdb; } else if (track && tdbIsComposite(track)) { struct trackDb *subTdb; for (subTdb=track->subtracks; subTdb != NULL; subTdb = subTdb->next) { if (subTdb->table && sameWord(subTdb->table, table)) { tdb = subTdb; break; } } } return(tdb); } char *unsplitTableName(char *table) /* Convert chr*_name to name */ { if (startsWith("chr", table)) { char *s = strrchr(table, '_'); if (s != NULL) { table = s + 1; } } return table; } static char *chopAtFirstDot(char *string) /* Terminate string at first '.' if found. Return string for convenience. */ { char *ptr = strchr(string, '.'); if (ptr != NULL) *ptr = '\0'; return string; } static struct hash *accessControlInit(char *db, struct sqlConnection *conn) /* Return a hash associating restricted table/track names in the given db/conn * with virtual hosts, or NULL if there is no tableAccessControl table. */ { struct hash *acHash = NULL; if (sqlTableExists(conn, "tableAccessControl")) { struct sqlResult *sr = NULL; char **row = NULL; acHash = newHash(8); char query[1024]; sqlSafef(query, sizeof query, "select name,host from tableAccessControl"); sr = sqlGetResult(conn, query); while ((row = sqlNextRow(sr)) != NULL) { struct slName *sln = slNameNew(chopAtFirstDot(row[1])); struct hashEl *hel = hashLookup(acHash, row[0]); if (hel == NULL) hashAdd(acHash, row[0], sln); else slAddHead(&(hel->val), sln); } sqlFreeResult(&sr); } return acHash; } static boolean accessControlDenied(struct hash *acHash, char *table) /* Return TRUE if table access is restricted to some host(s) other than * the one we're running on. */ { static char *currentHost = NULL; struct slName *enabledHosts = NULL; struct slName *sln = NULL; if (acHash == NULL) return FALSE; enabledHosts = (struct slName *)hashFindVal(acHash, table); if (enabledHosts == NULL) return FALSE; if (currentHost == NULL) { currentHost = cloneString(cgiServerName()); if (currentHost == NULL) { warn("accessControl: unable to determine current host"); return FALSE; } else chopAtFirstDot(currentHost); } for (sln = enabledHosts; sln != NULL; sln = sln->next) { if (sameString(currentHost, sln->name)) return FALSE; } return TRUE; } static void freeHelSlNameList(struct hashEl *hel) /* Helper function for hashTraverseEls, to free slNameList vals. */ { slNameFreeList(&(hel->val)); } static void accessControlFree(struct hash **pAcHash) /* Free up access control hash. */ { if (*pAcHash != NULL) { hashTraverseEls(*pAcHash, freeHelSlNameList); freeHash(pAcHash); } } struct slName *tablesForDb(char *db) /* Find tables associated with database. */ { boolean isGenomeDb = sameString(db, database); struct sqlConnection *conn = sqlConnect(db); struct slName *raw, *rawList = sqlListTables(conn); struct slName *cooked, *cookedList = NULL; struct hash *uniqHash = newHash(0); struct hash *accessCtlHash = accessControlInit(db, conn); sqlDisconnect(&conn); for (raw = rawList; raw != NULL; raw = raw->next) { if (isGenomeDb) { /* Deal with tables split across chromosomes. */ char *root = unsplitTableName(raw->name); if (accessControlDenied(accessCtlHash, root) || accessControlDenied(accessCtlHash, raw->name)) continue; if (!hashLookup(uniqHash, root)) { hashAdd(uniqHash, root, NULL); cooked = slNameNew(root); slAddHead(&cookedList, cooked); } } else { char dbTable[256]; if (accessControlDenied(accessCtlHash, raw->name)) continue; safef(dbTable, sizeof(dbTable), "%s.%s", db, raw->name); cooked = slNameNew(dbTable); slAddHead(&cookedList, cooked); } } hashFree(&uniqHash); accessControlFree(&accessCtlHash); slFreeList(&rawList); slSort(&cookedList, slNameCmp); return cookedList; } boolean htiIsPositional(struct hTableInfo *hti) /* Return TRUE if hti looks like it's from a positional table. */ { return isCustomTrack(hti->rootName) || ((hti->startField[0] && hti->endField[0]) && (hti->chromField[0] || sameString(hti->rootName, "gl"))); } char *showTableField(struct trackDb *track, char *varName, boolean useJoiner) /* Show table control and label. */ { struct slName *name, *nameList = NULL, *newList = NULL, *next = NULL; char *selTable; if (track == NULL) nameList = tablesForDb(findSelDb()); else nameList = tablesForTrack(track, useJoiner); /* filter out non-positional tables, chains and nets, and other untouchables */ for (name = nameList; name != NULL; name = next) { next = name->next; boolean isCt = isCustomTrack(name->name); if (strchr(name->name, '.')) continue; /* filter out anything starting with database, e.g. db.name (like Locus Variants) */ if (!isCt) /* all custom tracks are positional */ { struct hTableInfo *hti = hFindTableInfo(database, NULL, name->name); if (!hti) continue; /* filter out tables that don't exist anymore */ if (!htiIsPositional(hti) && (!track || (!startsWith("bam", track->type) && !startsWith("big", track->type))) ) continue; /* filter out non-positional, do not add it to new list */ } slAddHead(&newList,name); } slReverse(&newList); nameList = newList; /* Get currently selected table. If it isn't in our list * then revert to first in list. */ selTable = cartUsualString(cart, varName, nameList->name); if (!slNameInList(nameList, selTable)) selTable = nameList->name; /* Print out label and drop-down list. */ hPrintf("table: "); hPrintf("\n"); return selTable; } void importPage(struct sqlConnection *conn) /* Put up initial import page. */ { struct grp *selGroup; boolean isWig = FALSE, isPositional = FALSE, isMaf = FALSE, isBedGr = FALSE, isChromGraphCt = FALSE; struct hTableInfo *hti = NULL; cartWebStart(cart, database, "Import Table to Genome Graphs"); hPrintf("
", cartUsualString(cart, "formMethod", "POST")); cartSaveSession(cart); jsInit(); allJoiner = joinerRead("all.joiner"); initGroupsTracksTables(); hPrintf("\n"); /* Print group and track line. */ hPrintf("\n"); /* Print table line. */ hPrintf("\n"); if (curTrack == NULL) { struct trackDb *tdb = hTrackDbForTrack(database, curTable); struct trackDb *cTdb = hCompositeTrackDbForSubtrack(database, tdb); if (cTdb) curTrack = cTdb; else curTrack = tdb; isMaf = isMafTable(database, curTrack, curTable); } /* debug info hPrintf("\n"); */ hPrintf("\n"); hPrintf("
"); selGroup = showGroupField(hggGroup, onChangeGroupOrTrack(), conn, TRUE); nbSpaces(3); curTrack = showTrackField(selGroup, hggTrack, onChangeGroupOrTrack()); hPrintf("
"); curTable = showTableField(curTrack, hggTable, TRUE); if (strchr(curTable, '.') == NULL) /* In same database */ { hti = getHti(database, curTable); isPositional = htiIsPositional(hti); } isWig = isWiggle(database, curTable); isMaf = isMafTable(database, curTrack, curTable); isBedGr = isBedGraph(curTable); nbSpaces(1); if (isCustomTrack(curTable)) { isChromGraphCt = isChromGraph(curTrack); } hPrintf("
"); hPrintf("
\n"); hPrintf("Debug Info:
\n"); hPrintf("----------------------------
\n"); hPrintf("isWig: %d
\n", isWig); hPrintf("isPositional: %d
\n", isPositional); hPrintf("isMaf: %d
\n", isMaf); hPrintf("isBedGr: %d
\n", isBedGr); hPrintf("isChromGraphCt: %d
\n", isChromGraphCt); hPrintf("
"); hPrintf("
\n"); hPrintf("name of data set: "); cartMakeTextVar(cart, hggDataSetName, "", 16); hPrintf("
"); hPrintf("description: "); cartMakeTextVar(cart, hggDataSetDescription, "", 64); hPrintf("
\n"); hPrintf("display min value: "); cartMakeTextVar(cart, hggMinVal, "", 5); hPrintf(" max value: "); cartMakeTextVar(cart, hggMaxVal, "", 5); hPrintf("
\n"); hPrintf("label values: "); cartMakeTextVar(cart, hggLabelVals, "", 32); hPrintf("
\n"); hPrintf("draw connecting lines between markers separated by up to "); cartMakeIntVar(cart, hggMaxGapToFill, 25000000, 8); hPrintf(" bases.
"); hPrintf(" "); char *convertType = cartUsualString(cart, hggBedConvertType, hggBedDepth); jsTrackingVar(hggBedConvertTypeJs, convertType); if (isPositional && !isWig && !isMaf && !isBedGr && !isChromGraphCt) { hPrintf("conversion type: "); jsMakeTrackingRadioButton(hggBedConvertType, hggBedConvertTypeJs, hggBedDepth, convertType); hPrintf(" depth  "); jsMakeTrackingRadioButton(hggBedConvertType, hggBedConvertTypeJs, hggBedCoverage, convertType); hPrintf(" coverage"); hPrintf("
\n"); hPrintf(" "); } hPrintf("
\n"); hPrintf("Note: Loading some tables can take up to a minute. "); hPrintf("If you are importing more than one data set please give them "); hPrintf("different names. Only the most recent data set of a given name is "); hPrintf("kept. Otherwise data sets will be kept for at least 48 hours from "); hPrintf("last use. After that time you may have to import them again."); hPrintf("
\n"); hPrintf("
\n"); hPrintf(" "); cgiMakeButton(hggSubmitImport, "submit"); // TODO: should I add a cancel button? hPrintf("
\n"); /* Hidden form - for benefit of javascript. */ { static char *saveVars[] = { hggImport, hggGroup, hggTrack, hggTable, hggBedConvertType }; jsCreateHiddenForm(cart, getScriptName(), saveVars, ArraySize(saveVars)); } /* Put up section that describes table types allowed. */ webNewSection("Import table types"); hPrintf("%s", "

All tables that have positional information can be imported. " "This includes BED, PSL, wiggle, MAF, bedGraph and other standard types. " "You can also use data in custom tracks. The chromGraph custom tracks " "are not available for import selection because these are already in the format desired. " "All types will be converted to a chromGraph custom track with a window-size of 10,000 bases. " "


" ); webNewSection("Using the import page"); hPrintf( "To import a table or custom track, " "choose the group, track, and table from the drop-down lists, " "and then submit. The other controls on this form are optional, though " "filling them out will sometimes enhance the display. " "The controls for display min and " "max values and connecting lines can be set later via the configuration " "page as well. Here is a description of each control." "