source: orange/orange/misc/serverfiles.py @ 9669:165371b04b4a

Revision 9669:165371b04b4a, 28.6 KB checked in by anze <anze.staric@…>, 2 years ago (diff)

Moved content of Orange dir to package dir

Line 
1"""
2==============================
3Server files (``serverfiles``)
4==============================
5
6.. index:: server files
7
8Server files allows users to download files from a common
9repository residing on the Orange server. It was designed to simplify
10the download and updates of external data sources for Orange Genomics add-on.
11Furthermore, an authenticated user can also manage the repository files with
12this module.
13
14Orange server file repository was created to store large files that do not
15come with Orange installation, but may be required from the user when
16running specific Orange functions. A typical example is Orange Bioinformatics
17package, which relies on large data files storing genome information.
18These do not come pre-installed, but are rather downloaded from the server
19when needed and stored in the local repository. The module provides low-level
20functionality to manage these files, and is used by Orange modules to locate
21the files from the local repository and update/download them when and if needed.
22
23Each managed file is described by domain and the file name.
24Domains are like directories - a place where files are put in.
25
26Domain should consist of less than 255 alphanumeric ASCII characters, whereas
27filenames can be arbitary long and can contain any ASCII character (including
28"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
29domain and filenames. Files can be protected or not. Protected files can
30only be accessed by authenticated users
31
32Local file management
33=====================
34
35The files are saved under Orange's settings directory,
36subdirectory buffer/bigfiles. Each domain is a subdirectory.
37A corresponding info
38file bearing the same name and an extension ".info" is created
39with every download. Info files
40contain title, tags, size and date and time of the file.
41
42.. autofunction:: allinfo
43
44.. autofunction:: download
45
46.. autofunction:: info
47
48.. autofunction:: listdomains
49
50.. autofunction:: listfiles
51
52.. autofunction:: localpath
53
54.. autofunction:: localpath_download
55
56.. autofunction:: needs_update
57
58.. autofunction:: remove
59
60.. autofunction:: remove_domain
61
62.. autofunction:: search
63
64.. autofunction:: update
65
66
67Remote file management
68======================
69
70.. autoclass:: ServerFiles
71    :members:
72
73Examples
74========
75
76Listing local files, files from the repository and downloading all available files from domain "demo" (:download:`serverfiles1.py <code/serverfiles1.py>`).
77
78.. literalinclude:: code/serverfiles1.py
79
80A possible output (it depends on the current repository state)::
81
82    My files []
83    Repository files ['orngServerFiles.py', 'urllib2_file.py']
84    Downloading all files in domain 'test'
85    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
86    Downloading orngServerFiles.py
87    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
88    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
89    Downloading urllib2_file.py
90    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
91    My files after download ['urllib2_file.py', 'orngServerFiles.py']
92    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
93
94A domain with a simple file can be built as follows (:download:`serverfiles2.py <code/serverfiles2.py>`). Of course,
95the username and password should be valid.
96
97.. literalinclude:: code/serverfiles2.py
98
99A possible output::
100
101    Uploaded.
102    Non-authenticated users see: ['']
103    Authenticated users see: ['titanic.tab']
104    Non-authenticated users now see: ['titanic.tab']
105    orngServerFiles.py file info:
106    {'datetime': '2011-03-15 13:18:53.029000',
107     'size': '45112',
108     'tags': ['basic', 'data set'],
109     'title': 'A sample .tab file'}
110
111"""
112
113import sys
114import socket
115
116# timeout in seconds
117timeout = 120
118socket.setdefaulttimeout(timeout)
119
120import urllib
121import urllib2
122import base64
123
124from orngMisc import ConsoleProgressBar
125import time, threading
126
127import os
128import shutil
129import glob
130import datetime
131import tempfile
132
133#defserver = "localhost:9999/"
134defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
135
136def _parseFileInfo(fir, separ="|||||"):
137    """
138    Parses file info from server.
139    """
140    l= fir.split(separ)
141    fi = {}
142    fi["size"] = l[0]
143    fi["datetime"] = l[1]
144    fi["title"] = l[2]
145    fi["tags"] = l[3].split(";")
146    return fi
147
148def _open_file_info(fname): #no outer usage
149    f = open(fname, 'rt')
150    info = _parseFileInfo(f.read(), separ='\n')
151    f.close()
152    return info
153
154def _save_file_info(fname, info): #no outer usage
155    f = open(fname, 'wt')
156    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
157    f.close()
158
159def _parseList(fl):
160    return fl.split("|||||")
161
162def _parseAllFileInfo(afi):
163    separf = "[[[[["
164    separn = "====="
165    fis = afi.split(separf)
166    out = []
167    for entry in fis:
168        if entry != "":
169            name, info = entry.split(separn)
170            out.append((name, _parseFileInfo(info)))
171
172    return dict(out)
173
174def _create_path_for_file(target):
175    try:
176        os.makedirs(os.path.dirname(target))
177    except OSError:
178        pass
179
180def _create_path(target):
181    try:
182        os.makedirs(target)
183    except OSError:
184        pass
185
186def localpath(domain=None, filename=None):
187    """Return a path for the domain in the local repository. If
188    filename is given, return a path to corresponding file."""
189    import orngEnviron
190    if not domain:
191        return os.path.join(orngEnviron.directoryNames["bufferDir"],
192            "bigfiles")
193    if filename:
194        return os.path.join(orngEnviron.directoryNames["bufferDir"],
195            "bigfiles", domain, filename)
196    else:
197        return os.path.join(orngEnviron.directoryNames["bufferDir"],
198            "bigfiles", domain)
199
200class ServerFiles(object):
201    """
202    To work with the repository, you need to create an instance of
203    ServerFiles object. To access the repository as an authenticated user, a
204    username and password should be passed to the constructor. All password
205    protected operations and transfers are secured by SSL; this secures
206    both password and content.
207
208    Repository files are set as protected when first uploaded: only
209    authenticated users can see them. They need to be unprotected for
210    public use.
211    """
212
213    def __init__(self, username=None, password=None, server=None, access_code=None):
214        """
215        Creates a ServerFiles instance. Pass your username and password
216        to use the repository as an authenticated user. If you want to use
217        your access code (as an non-authenticated user), pass it also.
218        """
219        if not server:
220            server = defserver
221        self.server = server
222        self.secureroot = 'https://' + self.server + 'private/'
223        self.publicroot = 'http://' + self.server + 'public/'
224        self.username = username
225        self.password = password
226        self.access_code = access_code
227        self.searchinfo = None
228
229    def _getOpener(self, multipart=False):
230        opener = urllib2.build_opener()
231        return opener
232 
233    def upload(self, domain, filename, file, title="", tags=[]):
234        """ Uploads a file "file" to the domain where it is saved with filename
235        "filename". If file does not exist yet, set it as protected. Parameter
236        file can be a file handle open for reading or a file name.
237        """
238        if isinstance(file, basestring):
239            file = open(file, 'rb')
240
241        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
242        return self._open('upload', data)
243
244    def create_domain(self, domain):
245        """Create a server domain."""
246        return self._open('createdomain', { 'domain': domain })
247
248    def remove_domain(self, domain, force=False):
249        """Remove a domain. If force is True, domain is removed even
250        if it is not empty (contains files)."""
251        data = { 'domain': domain }
252        if force:
253            data['force'] = True
254        return self._open('removedomain', data)
255
256    def remove(self, domain, filename):
257        """Remove a file from the server repository."""
258        return self._open('remove', { 'domain': domain, 'filename': filename })
259
260    def unprotect(self, domain, filename):
261        """Put a file into public use."""
262        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
263
264    def protect(self, domain, filename, access_code="1"):
265        """Hide file from non-authenticated users. If an access code (string)
266        is passed, the file will be available to authenticated users and
267        non-authenticated users with that access code."""
268        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
269
270    def protection(self, domain, filename):
271        """Return file protection. Legend: "0" - public use,
272        "1" - for authenticated users only, anything else
273        represents a specific access code.
274        """
275        return self._open('protection', { 'domain': domain, 'filename': filename })
276   
277    def listfiles(self, domain):
278        """List all files in a repository domain."""
279        return _parseList(self._open('list', { 'domain': domain }))
280
281    def listdomains(self):
282        """List all domains on repository."""
283        return _parseList(self._open('listdomains', {}))
284
285    def downloadFH(self, *args, **kwargs):
286        """Return open file handle of requested file from the server repository given the domain and the filename."""
287        if self._authen(): return self.secdownloadFH(*args, **kwargs)
288        else: return self.pubdownloadFH(*args, **kwargs)
289
290    def download(self, domain, filename, target, callback=None):
291        """
292        Downloads file from the repository to a given target name. Callback
293        can be a function without arguments. It will be called once for each
294        downloaded percent of file: 100 times for the whole file.
295        """
296        _create_path_for_file(target)
297
298        fdown = self.downloadFH(domain, filename)
299        size = int(fdown.headers.getheader('content-length'))
300
301        f = tempfile.TemporaryFile()
302 
303        chunksize = 1024*8
304        lastchunkreport= 0.0001
305
306        readb = 0
307        while 1:
308            buf = fdown.read(chunksize)
309            readb += len(buf)
310
311            while float(readb)/size > lastchunkreport+0.01:
312                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
313                lastchunkreport += 0.01
314                if callback:
315                    callback()
316            if not buf:
317                break
318            f.write(buf)
319
320        fdown.close()
321        f.seek(0)
322
323        shutil.copyfileobj(f, open(target, "wb"))
324
325        if callback:
326            callback()
327
328    def _searchinfo(self):
329        domains = self.listdomains()
330        infos = {}
331        for dom in domains:
332            dominfo = self.allinfo(dom)
333            for a,b in dominfo.items():
334                infos[(dom, a)] = b
335        return infos
336
337    def search(self, sstrings, **kwargs):
338        """
339        Search for files on the repository where all substrings in a list
340        are contained in at least one choosen field (tag, title, name). Return
341        a list of tuples: first tuple element is the file's domain, second its
342        name. As for now the search is performed locally, therefore
343        information on files in repository is transfered on first call of
344        this function.
345        """
346        if not self.searchinfo:
347            self.searchinfo = self._searchinfo()
348        return _search(self.searchinfo, sstrings, **kwargs)
349
350    def info(self, domain, filename):
351        """Return a dictionary containing repository file info.
352        Keys: title, tags, size, datetime."""
353        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
354
355    def downloadFH(self, domain, filename):
356        """Return a file handle to the file that we would like to download."""
357        return self._handle('download', { 'domain': domain, 'filename': filename })
358
359    def list(self, domain):
360        return _parseList(self._open('list', { 'domain': domain }))
361
362    def listdomains(self):
363        """List all domains on repository."""
364        return _parseList(self._open('listdomains', {}))
365
366    def allinfo(self, domain):
367        """Go through all accessible files in a given domain and return a
368        dictionary, where key is file's name and value its info.
369        """
370        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
371
372    def index(self):
373        return self._open('index', {})
374
375    def _authen(self):
376        """
377        Did the user choose authentication?
378        """
379        if self.username and self.password:
380            return True
381        else:
382            return False
383
384    def _server_request(self, root, command, data, repeat=2):
385        def do():
386            opener = self._getOpener()
387           
388            if data:
389                if command == "upload":
390                    # Need to use poster to handle multipart post
391                    try:
392                        import poster.streaminghttp as psh
393                        import poster.encode
394                    except ImportError:
395                        raise ImportError("You need to install 'poster' (http://pypi.python.org/pypi/poster) to be able to upload files.")
396               
397                    handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
398                    opener = urllib2.build_opener(*handlers)
399                    datagen, headers = poster.encode.multipart_encode(data)
400                    request = urllib2.Request(root+command, datagen, headers)
401                else:
402                    request = urllib2.Request(root+command, urllib.urlencode(data))
403            else:
404                request = urllib2.Request(root+command)
405
406            #directy add authorization headers
407            if self._authen():
408                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
409                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
410           
411            return opener.open(request)
412               
413        if repeat <= 0:
414            return do()
415        else:
416            try:
417                return do()
418            except:
419                return self._server_request(root, command, data, repeat=repeat-1)
420   
421    def _handle(self, command, data):
422        data2 = self._addAccessCode(data)
423        addr = self.publicroot
424        if self._authen():
425            addr = self.secureroot
426        return self._server_request(addr, command, data)
427
428    def _open(self, command, data):
429        return self._handle(command, data).read()
430
431    def _addAccessCode(self, data):
432        if self.access_code != None:
433            data = data.copy()
434            data["access_code"] = self.access_code
435        return data
436
437def download(domain, filename, serverfiles=None, callback=None, 
438    extract=True, verbose=True):
439    """Downloads file from the repository to local orange installation.
440    To download files as an authenticated user you should also pass an
441    instance of ServerFiles class. Callback can be a function without
442    arguments. It will be called once for each downloaded percent of
443    file: 100 times for the whole file."""
444
445    if not serverfiles:
446        serverfiles = ServerFiles()
447
448    info = serverfiles.info(domain, filename)
449    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
450    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
451    target = localpath(domain, filename)
452    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
453    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
454   
455    #file saved, now save info file
456
457    _save_file_info(target + '.info', info)
458   
459    if extract:
460        import tarfile, gzip, shutil
461        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
462            f = tarfile.open(target + ".tmp")
463            f.extractall(localpath(domain))
464            shutil.copyfile(target + ".tmp", target)
465        elif filename.endswith(".tar.gz"):
466            f = tarfile.open(target + ".tmp")
467            try:
468                os.mkdir(target)
469            except Exception:
470                pass
471            f.extractall(target)
472        elif specialtags.get("#compression") == "gz":
473            f = gzip.open(target + ".tmp")
474            shutil.copyfileobj(f, open(target, "wb"))
475        f.close()
476        os.remove(target + ".tmp")
477
478    if type(callback) == DownloadProgress:
479        callback.finish()
480
481def localpath_download(domain, filename, **kwargs):
482    """
483    Return local path for the given domain and file. If file does not exist,
484    download it. Additional arguments are passed to the :obj:`download` function.
485    """
486    pathname = localpath(domain, filename)
487    if not os.path.exists(pathname):
488        download(domain, filename, **kwargs)
489    return pathname
490
491def listfiles(domain):
492    """List all files from a domain in a local repository."""
493    dir = localpath(domain)
494    try:
495        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
496    except:
497        files = []
498    okfiles = []
499
500    for file in files:
501        #if file to exists without info
502        if os.path.exists(os.path.join(dir,file[:-5])):
503            #check info format - needs to be valid
504            try:
505                _open_file_info(os.path.join(dir,file))
506                okfiles.append(file[:-5])
507            except:
508                pass
509
510    return okfiles
511
512def remove(domain, filename):
513    """Remove a file from local repository."""
514    filename = localpath(domain, filename)
515    import shutil
516   
517    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
518    todelete = [filename, filename + ".info"] 
519    if "#files" in specialtags:
520        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
521#    print todelete
522    for path in todelete:
523        try:
524            if os.path.isdir(path):
525                shutil.rmtree(path)
526            elif os.path.isfile(path):
527                os.remove(path)
528        except OSError, ex:
529            print "Failed to delete", path, "due to:", ex
530   
531def remove_domain(domain, force=False):
532    """Remove a domain. If force is True, domain is removed even
533    if it is not empty (contains files)."""
534    directory = localpath(domain)
535    if force:
536        import shutil
537        shutil.rmtree(directory)
538    else:
539        os.rmdir(directory)
540
541def listdomains():
542    """List all file domains in the local repository."""
543    dir = localpath()
544    _create_path(dir)
545    files = [ a for a in os.listdir(dir) ]
546    ok = []
547    for file in files:
548        if os.path.isdir(os.path.join(dir, file)):
549            ok.append(file)
550    return ok
551
552def info(domain, filename):
553    """Returns info of a file in a local repository."""
554    target = localpath(domain, filename)
555    return _open_file_info(target + '.info')
556
557def allinfo(domain):
558    """Goes through all files in a domain on a local repository and returns a
559    dictionary, where keys are names of the files and values are their
560    information."""
561    files = listfiles(domain)
562    dic = {}
563    for filename in files:
564        target = localpath(domain, filename)
565        dic[filename] = info(domain, target)
566    return dic
567
568def needs_update(domain, filename, serverfiles=None):
569    """True if a file does not exist in the local repository
570    or if there is a newer version on the server."""
571    if serverfiles == None: serverfiles = ServerFiles()
572    if filename not in listfiles(domain):
573        return True
574    dt_fmt = "%Y-%m-%d %H:%M:%S"
575    dt_local = datetime.datetime.strptime(
576        info(domain, filename)["datetime"][:19], dt_fmt)
577    dt_server = datetime.datetime.strptime(
578        serverfiles.info(domain, filename)["datetime"][:19], dt_fmt)
579    return dt_server > dt_local
580
581def update(domain, filename, serverfiles=None, **kwargs):
582    """Downloads the corresponding file from the server and places it in
583    the local repository, but only if the server copy of the file is newer
584    or the local copy does not exist. An optional  :class:`ServerFiles` object
585    can be passed for authenticated access.
586    """
587    if serverfiles == None: serverfiles = ServerFiles()
588    if needs_update(domain, filename, serverfiles=serverfiles):
589        download(domain, filename, serverfiles=serverfiles, **kwargs)
590       
591def _searchinfo():
592    domains = listdomains()
593    infos = {}
594    for dom in domains:
595        dominfo = allinfo(dom)
596        for a,b in dominfo.items():
597            infos[(dom, a)] = b
598    return infos
599
600def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
601    """
602    sstrings contain a list of search strings
603    """
604    found = []
605
606    for (dom,fn),info in si.items():
607        target = ""
608        if inTag: target += " ".join(info['tags'])
609        if inTitle: target += info['title']
610        if inName: target += fn
611        if not caseSensitive: target = target.lower()
612
613        match = True
614        for s in sstrings:
615            if not caseSensitive:
616                s = s.lower()
617            if s not in target:
618                match= False
619                break
620               
621        if match:
622            found.append((dom,fn))   
623       
624    return found
625
626def search(sstrings, **kwargs):
627    """Search for files in the local repository where all substrings in a list
628    are contained in at least one chosen field (tag, title, name). Return a
629    list of tuples: first tuple element is the domain of the file, second
630    its name."""
631    si = _searchinfo()
632    return _search(si, sstrings, **kwargs)
633
634class DownloadProgress(ConsoleProgressBar):
635    redirect = None
636    lock = threading.RLock()
637    def sizeof_fmt(num):
638        for x in ['bytes','KB','MB','GB','TB']:
639            if num < 1024.0:
640                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
641            num /= 1024.0
642           
643    def __init__(self, filename, size):
644        print "Downloading", filename
645        ConsoleProgressBar.__init__(self, "progress:", 20)
646        self.size = size
647        self.starttime = time.time()
648        self.speed = 0.0
649
650    def sizeof_fmt(self, num):
651        for x in ['bytes','KB','MB','GB','TB']:
652            if num < 1024.0:
653                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
654            num /= 1024.0
655
656    def getstring(self):
657        speed = int(self.state * self.size / 100.0 / (time.time() - self.starttime))
658        eta = (100 - self.state) * self.size / 100.0 / speed
659        return ConsoleProgressBar.getstring(self) + %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size), self.sizeof_fmt(speed), eta/60, eta%60)
660       
661    def __call__(self, *args, **kwargs):
662        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
663        if self.redirect:
664            self.redirect(self.state)
665        return ret
666   
667    class RedirectContext(object):
668        def __enter__(self):
669            DownloadProgress.lock.acquire()
670            return DownloadProgress
671       
672        def __exit__(self, ex_type, value, tb):
673            DownloadProgress.redirect = None
674            DownloadProgress.lock.release()
675            return False
676       
677    @classmethod
678    def setredirect(cls, redirect):
679        cls.redirect = staticmethod(redirect)
680        return cls.RedirectContext()
681   
682    @classmethod
683    def __enter__(cls):
684        cls.lock.acquire()
685        return cls
686   
687    @classmethod
688    def __exit__(cls, exc_type, exc_value, traceback):
689        cls.lock.release()
690        return False
691
692def consoleupdate(domains=None, searchstr="essential"):
693    domains = domains or listdomains()
694    sf = ServerFiles()
695    info = dict((d, sf.allinfo(d)) for d in domains)
696    def searchmenu():
697        def printmenu():
698            print "\tSearch tags:", search
699            print "\t1. Add tag."
700            print "\t2. Clear tags."
701            print "\t0. Return to main menu."
702            return raw_input("\tSelect option:")
703        search = searchstr
704        while True:
705            response = printmenu().strip()
706            if response == "1":
707                search += " " + raw_input("\tType new tag/tags:")
708            elif response == "2":
709                search = ""
710            elif response == "0":
711                break
712            else:
713                print "\tUnknown option!"
714        return search
715
716    def filemenu(searchstr=""):
717        files = [None]
718        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
719            print "\t%i." % (i + 1), info[dom][file]["title"]
720            files.append((dom, file))
721        print "\t0. Return to main menu."
722        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
723        while True:
724            response = raw_input("\tAction:").strip()
725            if response == "0":
726                break
727            try:
728                action, num = response.split(None, 1)
729                num = int(num)
730            except Exception, ex:
731                print "Unknown option!"
732                continue
733            try:
734                if action.lower() == "d":
735                    download(*(files[num]))
736                    print "\tSuccsessfully downloaded", files[num][-1]
737            except Exception, ex:
738                print "Error occured!", ex
739
740    def printmenu():
741        print "Update database main menu:"
742        print "1. Enter search tags (refine search)."
743        print "2. Print matching available files."
744        print "3. Print all available files."
745        print "4. Update all local files."
746        print "0. Exit."
747        return raw_input("Select option:")
748   
749    while True:
750        try:
751            response = printmenu().strip()
752            if response == "1":
753                searchstr = searchmenu()
754            elif response == "2":
755                filemenu(searchstr)
756            elif response == "3":
757                filemenu("")
758            elif response == "4":
759                update_local_files()
760            elif response == "0":
761                break
762            else:
763                print "Unknown option!"
764        except Exception, ex:
765            print "Error occured:", ex
766
767def update_local_files(verbose=True):
768    sf = ServerFiles()
769    for domain, filename in search(""):
770        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
771        if not uptodate:
772            download(domain, filename, sf)
773        if verbose:
774            print filename, "Ok" if uptodate else "Updated"
775
776def update_by_tags(tags=["essential"], domains=[], verbose=True):
777    sf = ServerFiles()
778    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
779        if domains and domain not in domain:
780            continue
781        if os.path.exists(localpath(domain, filename)+".info"):
782            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
783        else:
784            uptodate = False
785        if not uptodate:
786            download(domain, filename, sf)
787        if verbose:
788            print filename, "Ok" if uptodate else "Updated"
789           
790def _example(myusername, mypassword):
791
792    locallist = listfiles('test')
793    for l in locallist:
794        print info('test', l)
795
796    s = ServerFiles()
797
798    print "testing connection - public"
799    print "AN", s.index()
800
801    #login as an authenticated user
802    s = ServerFiles(username=myusername, password=mypassword)
803   
804    """
805    print "Server search 1"
806    import time
807    t = time.time()
808    print s.search(["rat"])
809    print time.time() - t
810
811    t = time.time()
812    print s.search(["human", "ke"])
813    print time.time() - t
814    """
815
816    print "testing connection - private"
817    print "AN", s.index()
818
819    #create domain
820    try: 
821        s.create_domain("test") 
822    except:
823        print "Failed to create the domain"
824        pass
825
826    files = s.listfiles('test')
827    print "Files in test", files
828
829    print "uploading"
830
831    #upload this file - save it by a different name
832    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
833    #make it public
834    s.unprotect('test', 'osf-test.py')
835
836    #login anonymously
837    s = ServerFiles()
838
839    #list files in the domain "test"
840    files = s.listfiles('test')
841    print "ALL FILES:", files
842
843    for f in files:
844        fi = s.info('test', f) 
845        print "--------------------------------------", f
846        print "INFO", fi
847        print s.downloadFH('test', f).read()[:100] #show first 100 characters
848        print "--------------------------------------"
849
850    #login as an authenticated user
851    s = ServerFiles(username=myusername, password=mypassword)
852
853    print s.listdomains()
854
855    s.remove('test', 'osf-test.py')
856
857    s = ServerFiles()
858
859    print s.listdomains()
860
861
862if __name__ == '__main__':
863    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.