- [ ] implement google drive uploading function - put to utils.py/config.py
- Paper writing:
- new folder:
- src/paper/1-chipnexus-data-stats.ipynb
- src/paper/2-model-eval.ipynb
- src/paper/3-modisco-motifs.ipynb
- new folder:
- Paper writing:
In [1]:
from basepair.imports import *
https://developers.google.com/drive/api/v3/folder -> refer to a folder
https://github.com/gsuitedevs/PyDrive
save a figure to google drive.¶
Saves png and pdf versions to tmp and uploads them to google drive
def gdrive_save_fig(fig, name, parent_id):
pass
In [136]:
"""Tools to directly upload files to google drive
Requires pydrive to be installed (https://github.com/gsuitedevs/PyDrive)
To setup the credentials, do the following tutorial:
https://gsuitedevs.github.io/PyDrive/docs/build/html/quickstart.html#authentication
Once you get client_secrets.json,
run the following code:
```python
import os
from pydrive.auth import GoogleAuth
gauth = GoogleAuth()
gauth.LoadClientConfigFile("/users/avsec/.gdrive/client_secrets.json") # path to client_secrets.json
gauth.LocalWebserverAuth() # Creates local webserver and auto handles authentication.
gauth.SaveCredentialsFile(os.path.expanduser("~/.gdrive.creds.txt"))
````
"""
CHIPNEXUS_RAW_ID = "1oJDXr2vbP89WS4Dp3mYli-aJi7Q1oZbR"
def get_drive(credentials_file="~/.gdrive.creds.txt"):
import os
from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
import logging
logging.getLogger('googleapiclient.discovery').setLevel(logging.CRITICAL)
logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR)
gauth = GoogleAuth()
# Try to load saved client credentials
gauth.LoadCredentialsFile(os.path.expanduser(credentials_file))
if gauth.credentials is None:
# Authenticate if they're not there
gauth.LocalWebserverAuth()
elif gauth.access_token_expired:
# Refresh them if expired
gauth.Refresh()
else:
# Initialize the saved creds
gauth.Authorize()
drive = GoogleDrive(gauth)
return drive
def list_folder(drive, parent):
filelist=[]
file_list = drive.ListFile({'q': "'%s' in parents and trashed=false" % parent}).GetList()
for f in file_list:
if f['mimeType']=='application/vnd.google-apps.folder': # if folder
filelist.append({"id":f['id'],"title":f['title'],"type":"dir", "list": list_folder(drive, f['id'])})
else:
filelist.append({"title":f['title'], "type": "file", "link":f['alternateLink'], "id": f['id']})
return filelist
def get_file(drive, title, parent_id):
"""Retrieve the google drive file
If it doesn't exist, create it. Also works
when title is nested: folder/subfolder/file
"""
files = list_folder(drive, parent_id)
assert "//" not in title
if "/" in title:
pdir, new_file = title.split("/", 1)
for f in files:
if f['type'] == "dir" and pdir == f['title']:
# folder exists
return get_file(drive, new_file, f['id'])
# folder doesn't exist. create one
pdir_id = drive.CreateFile({'title': pdir,
'parents': [{"id": parent_id}],
'mimeType': 'application/vnd.google-apps.folder'})
pdir_id.Upload()
return get_file(drive, new_file, pdir_id['id'])
for f in files:
if title == f['title']:
return drive.CreateFile({"id": f['id']})
return drive.CreateFile({'title': title, 'parents': [{"id": parent_id}]})
def gdrive_upload_file(file, title, parent_id=CHIPNEXUS_RAW_ID):
"""Upload a file to google drive
Args:
file: file path
title: file name in google drive
parent_id: folder parent id
"""
drive = get_drive()
file1 = get_file(drive, title, parent_id)
file1.SetContentFile(file)
file1.Upload()
def gdrive_save_fig(fig, name, parent_id=CHIPNEXUS_RAW_ID):
"""Upload matplot figure to google drive
Args:
fig: matplotlib.figure
name: file name (without suffix)
parent_id: folder parent id
"""
import tempfile
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
with tempfile.TemporaryDirectory() as td:
tdp = Path(td)
fig.savefig(tdp / "fig.png", dpi=300)
fig.savefig(tdp / "fig.pdf")
gdrive_upload_file(str(tdp / "fig.png"), f"{name}.png", parent_id)
gdrive_upload_file(str(tdp / "fig.pdf"), f"{name}.pdf", parent_id)
In [126]:
file = "test/dsa/file.txt"
In [123]:
if "/" in file:
pdir, file = "test/dsa/file.txt".split("/", 1)
In [106]:
drive = get_drive()
In [127]:
list_folder(drive, parent_id)
Out[127]:
In [142]:
fig = plt.figure()
plt.plot(np.arange(10), "-o");
In [140]:
gdrive_save_fig(fig, "test/graph")
In [78]:
list_folder(drive, parent_id)
Out[78]:
In [131]:
gdrive_upload_file("/tmp/text.txt", "test_folder/text.txt")
In [135]:
gdrive_upload_file("/tmp/text.txt", "new_folder/nested/text.txt")
In [133]:
!echo "hey4" > /tmp/text.txt
In [47]:
file = "/tmp/test.txt"
In [6]:
parent_id = "1oJDXr2vbP89WS4Dp3mYli-aJi7Q1oZbR"
In [30]:
drive = get_drive()
In [56]:
file1 = drive.CreateFile({'title': 'Hello.txt', 'parents': [{"id": parent_id}]}) # Create GoogleDriveFile instance with title 'Hello.txt'.
file1.SetContentString('Hello World!') # Set content of the file from given string.
file1.Upload()
In [44]:
file1.SetContentFile?