source: orange/Orange/utils/serverfiles.py @ 11438:81793ae1f7d1

Revision 11438:81793ae1f7d1, 28.8 KB checked in by Ales Erjavec <ales.erjavec@…>, 13 months ago (diff)

Fixed a 'ZeroDivisionError' when downloading a zero lenght file.

RevLine 
[8042]1"""
2==============================
3Server files (``serverfiles``)
4==============================
5
6.. index:: server files
7
8Server files allows users to download files from a common
9repository residing on the Orange server. It was designed to simplify
10the download and updates of external data sources for Orange Genomics add-on.
11Furthermore, an authenticated user can also manage the repository files with
12this module.
13
14Orange server file repository was created to store large files that do not
15come with Orange installation, but may be required from the user when
16running specific Orange functions. A typical example is Orange Bioinformatics
17package, which relies on large data files storing genome information.
18These do not come pre-installed, but are rather downloaded from the server
19when needed and stored in the local repository. The module provides low-level
20functionality to manage these files, and is used by Orange modules to locate
21the files from the local repository and update/download them when and if needed.
22
23Each managed file is described by domain and the file name.
24Domains are like directories - a place where files are put in.
25
26Domain should consist of less than 255 alphanumeric ASCII characters, whereas
27filenames can be arbitary long and can contain any ASCII character (including
28"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
29domain and filenames. Files can be protected or not. Protected files can
30only be accessed by authenticated users
31
32Local file management
33=====================
34
35The files are saved under Orange's settings directory,
36subdirectory buffer/bigfiles. Each domain is a subdirectory.
37A corresponding info
38file bearing the same name and an extension ".info" is created
39with every download. Info files
40contain title, tags, size and date and time of the file.
41
42.. autofunction:: allinfo
43
44.. autofunction:: download
45
46.. autofunction:: info
47
48.. autofunction:: listdomains
49
50.. autofunction:: listfiles
51
52.. autofunction:: localpath
53
54.. autofunction:: localpath_download
55
56.. autofunction:: needs_update
57
58.. autofunction:: remove
59
60.. autofunction:: remove_domain
61
62.. autofunction:: search
63
64.. autofunction:: update
65
66
67Remote file management
68======================
69
70.. autoclass:: ServerFiles
71    :members:
72
73Examples
74========
75
[9349]76Listing local files, files from the repository and downloading all available files from domain "demo" (:download:`serverfiles1.py <code/serverfiles1.py>`).
[8042]77
78.. literalinclude:: code/serverfiles1.py
79
80A possible output (it depends on the current repository state)::
81
82    My files []
83    Repository files ['orngServerFiles.py', 'urllib2_file.py']
84    Downloading all files in domain 'test'
85    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
86    Downloading orngServerFiles.py
87    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
88    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
89    Downloading urllib2_file.py
90    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
91    My files after download ['urllib2_file.py', 'orngServerFiles.py']
92    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
93
[9349]94A domain with a simple file can be built as follows (:download:`serverfiles2.py <code/serverfiles2.py>`). Of course,
[8042]95the username and password should be valid.
96
97.. literalinclude:: code/serverfiles2.py
98
99A possible output::
100
101    Uploaded.
102    Non-authenticated users see: ['']
103    Authenticated users see: ['titanic.tab']
104    Non-authenticated users now see: ['titanic.tab']
105    orngServerFiles.py file info:
106    {'datetime': '2011-03-15 13:18:53.029000',
107     'size': '45112',
108     'tags': ['basic', 'data set'],
109     'title': 'A sample .tab file'}
110
111"""
112
113import sys
114import socket
115
116# timeout in seconds
117timeout = 120
118socket.setdefaulttimeout(timeout)
119
[9370]120import urllib
[8042]121import urllib2
122import base64
123
[10580]124from Orange.utils import ConsoleProgressBar
[8042]125import time, threading
126
127import os
128import shutil
129import glob
130import datetime
131import tempfile
132
133#defserver = "localhost:9999/"
134defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
135
136def _parseFileInfo(fir, separ="|||||"):
137    """
138    Parses file info from server.
139    """
140    l= fir.split(separ)
141    fi = {}
142    fi["size"] = l[0]
143    fi["datetime"] = l[1]
144    fi["title"] = l[2]
145    fi["tags"] = l[3].split(";")
146    return fi
147
148def _open_file_info(fname): #no outer usage
149    f = open(fname, 'rt')
150    info = _parseFileInfo(f.read(), separ='\n')
151    f.close()
152    return info
153
154def _save_file_info(fname, info): #no outer usage
155    f = open(fname, 'wt')
156    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
157    f.close()
158
159def _parseList(fl):
160    return fl.split("|||||")
161
162def _parseAllFileInfo(afi):
163    separf = "[[[[["
164    separn = "====="
165    fis = afi.split(separf)
166    out = []
167    for entry in fis:
168        if entry != "":
169            name, info = entry.split(separn)
170            out.append((name, _parseFileInfo(info)))
171
172    return dict(out)
173
174def _create_path_for_file(target):
175    try:
176        os.makedirs(os.path.dirname(target))
177    except OSError:
178        pass
179
180def _create_path(target):
181    try:
182        os.makedirs(target)
183    except OSError:
184        pass
185
186def localpath(domain=None, filename=None):
187    """Return a path for the domain in the local repository. If
188    filename is given, return a path to corresponding file."""
189    import orngEnviron
190    if not domain:
191        return os.path.join(orngEnviron.directoryNames["bufferDir"],
192            "bigfiles")
193    if filename:
194        return os.path.join(orngEnviron.directoryNames["bufferDir"],
195            "bigfiles", domain, filename)
196    else:
197        return os.path.join(orngEnviron.directoryNames["bufferDir"],
198            "bigfiles", domain)
199
200class ServerFiles(object):
201    """
202    To work with the repository, you need to create an instance of
203    ServerFiles object. To access the repository as an authenticated user, a
204    username and password should be passed to the constructor. All password
205    protected operations and transfers are secured by SSL; this secures
206    both password and content.
207
208    Repository files are set as protected when first uploaded: only
209    authenticated users can see them. They need to be unprotected for
210    public use.
211    """
212
213    def __init__(self, username=None, password=None, server=None, access_code=None):
214        """
215        Creates a ServerFiles instance. Pass your username and password
216        to use the repository as an authenticated user. If you want to use
217        your access code (as an non-authenticated user), pass it also.
218        """
219        if not server:
220            server = defserver
221        self.server = server
222        self.secureroot = 'https://' + self.server + 'private/'
223        self.publicroot = 'http://' + self.server + 'public/'
224        self.username = username
225        self.password = password
226        self.access_code = access_code
227        self.searchinfo = None
228
[9370]229    def _getOpener(self, multipart=False):
[8042]230        opener = urllib2.build_opener()
231        return opener
232 
233    def upload(self, domain, filename, file, title="", tags=[]):
234        """ Uploads a file "file" to the domain where it is saved with filename
235        "filename". If file does not exist yet, set it as protected. Parameter
236        file can be a file handle open for reading or a file name.
237        """
238        if isinstance(file, basestring):
239            file = open(file, 'rb')
240
241        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
242        return self._open('upload', data)
243
244    def create_domain(self, domain):
245        """Create a server domain."""
246        return self._open('createdomain', { 'domain': domain })
247
248    def remove_domain(self, domain, force=False):
249        """Remove a domain. If force is True, domain is removed even
250        if it is not empty (contains files)."""
251        data = { 'domain': domain }
252        if force:
253            data['force'] = True
254        return self._open('removedomain', data)
255
256    def remove(self, domain, filename):
257        """Remove a file from the server repository."""
258        return self._open('remove', { 'domain': domain, 'filename': filename })
259
260    def unprotect(self, domain, filename):
261        """Put a file into public use."""
262        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
263
264    def protect(self, domain, filename, access_code="1"):
265        """Hide file from non-authenticated users. If an access code (string)
266        is passed, the file will be available to authenticated users and
267        non-authenticated users with that access code."""
268        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
269
270    def protection(self, domain, filename):
271        """Return file protection. Legend: "0" - public use,
272        "1" - for authenticated users only, anything else
273        represents a specific access code.
274        """
275        return self._open('protection', { 'domain': domain, 'filename': filename })
276   
277    def listfiles(self, domain):
278        """List all files in a repository domain."""
279        return _parseList(self._open('list', { 'domain': domain }))
280
281    def listdomains(self):
282        """List all domains on repository."""
283        return _parseList(self._open('listdomains', {}))
284
285    def downloadFH(self, *args, **kwargs):
286        """Return open file handle of requested file from the server repository given the domain and the filename."""
287        if self._authen(): return self.secdownloadFH(*args, **kwargs)
288        else: return self.pubdownloadFH(*args, **kwargs)
289
290    def download(self, domain, filename, target, callback=None):
291        """
292        Downloads file from the repository to a given target name. Callback
293        can be a function without arguments. It will be called once for each
294        downloaded percent of file: 100 times for the whole file.
295        """
296        _create_path_for_file(target)
297
298        fdown = self.downloadFH(domain, filename)
299        size = int(fdown.headers.getheader('content-length'))
300
301        f = tempfile.TemporaryFile()
302 
303        chunksize = 1024*8
304        lastchunkreport= 0.0001
305
306        readb = 0
[11438]307        # in case size == 0 skip the loop
308        while size > 0:
[8042]309            buf = fdown.read(chunksize)
310            readb += len(buf)
311
[11438]312            while float(readb) / size > lastchunkreport+0.01:
[8042]313                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
314                lastchunkreport += 0.01
315                if callback:
316                    callback()
317            if not buf:
318                break
319            f.write(buf)
320
321        fdown.close()
322        f.seek(0)
323
324        shutil.copyfileobj(f, open(target, "wb"))
325
326        if callback:
327            callback()
328
329    def _searchinfo(self):
330        domains = self.listdomains()
331        infos = {}
332        for dom in domains:
333            dominfo = self.allinfo(dom)
334            for a,b in dominfo.items():
335                infos[(dom, a)] = b
336        return infos
337
338    def search(self, sstrings, **kwargs):
339        """
340        Search for files on the repository where all substrings in a list
341        are contained in at least one choosen field (tag, title, name). Return
342        a list of tuples: first tuple element is the file's domain, second its
343        name. As for now the search is performed locally, therefore
344        information on files in repository is transfered on first call of
345        this function.
346        """
347        if not self.searchinfo:
348            self.searchinfo = self._searchinfo()
349        return _search(self.searchinfo, sstrings, **kwargs)
350
351    def info(self, domain, filename):
352        """Return a dictionary containing repository file info.
353        Keys: title, tags, size, datetime."""
354        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
355
356    def downloadFH(self, domain, filename):
357        """Return a file handle to the file that we would like to download."""
358        return self._handle('download', { 'domain': domain, 'filename': filename })
359
360    def list(self, domain):
361        return _parseList(self._open('list', { 'domain': domain }))
362
363    def listdomains(self):
364        """List all domains on repository."""
365        return _parseList(self._open('listdomains', {}))
366
367    def allinfo(self, domain):
368        """Go through all accessible files in a given domain and return a
369        dictionary, where key is file's name and value its info.
370        """
371        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
372
373    def index(self):
374        return self._open('index', {})
375
376    def _authen(self):
377        """
378        Did the user choose authentication?
379        """
380        if self.username and self.password:
381            return True
382        else:
383            return False
384
385    def _server_request(self, root, command, data, repeat=2):
386        def do():
387            opener = self._getOpener()
[9370]388           
[8042]389            if data:
[9370]390                if command == "upload":
391                    # Need to use poster to handle multipart post
392                    try:
393                        import poster.streaminghttp as psh
394                        import poster.encode
395                    except ImportError:
396                        raise ImportError("You need to install 'poster' (http://pypi.python.org/pypi/poster) to be able to upload files.")
397               
398                    handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
399                    opener = urllib2.build_opener(*handlers)
400                    datagen, headers = poster.encode.multipart_encode(data)
401                    request = urllib2.Request(root+command, datagen, headers)
402                else:
403                    request = urllib2.Request(root+command, urllib.urlencode(data))
[8042]404            else:
405                request = urllib2.Request(root+command)
406
407            #directy add authorization headers
408            if self._authen():
409                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
410                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
411           
412            return opener.open(request)
[9370]413               
[8042]414        if repeat <= 0:
415            return do()
416        else:
417            try:
418                return do()
419            except:
420                return self._server_request(root, command, data, repeat=repeat-1)
421   
422    def _handle(self, command, data):
423        data2 = self._addAccessCode(data)
424        addr = self.publicroot
425        if self._authen():
426            addr = self.secureroot
427        return self._server_request(addr, command, data)
428
429    def _open(self, command, data):
430        return self._handle(command, data).read()
431
432    def _addAccessCode(self, data):
433        if self.access_code != None:
434            data = data.copy()
435            data["access_code"] = self.access_code
436        return data
437
438def download(domain, filename, serverfiles=None, callback=None, 
439    extract=True, verbose=True):
440    """Downloads file from the repository to local orange installation.
441    To download files as an authenticated user you should also pass an
442    instance of ServerFiles class. Callback can be a function without
443    arguments. It will be called once for each downloaded percent of
444    file: 100 times for the whole file."""
445
446    if not serverfiles:
447        serverfiles = ServerFiles()
448
449    info = serverfiles.info(domain, filename)
450    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
451    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
452    target = localpath(domain, filename)
453    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
454    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
455   
456    #file saved, now save info file
457
458    _save_file_info(target + '.info', info)
459   
460    if extract:
461        import tarfile, gzip, shutil
462        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
463            f = tarfile.open(target + ".tmp")
464            f.extractall(localpath(domain))
465            shutil.copyfile(target + ".tmp", target)
[8997]466        elif filename.endswith(".tar.gz"):
[8042]467            f = tarfile.open(target + ".tmp")
468            try:
469                os.mkdir(target)
470            except Exception:
471                pass
472            f.extractall(target)
473        elif specialtags.get("#compression") == "gz":
474            f = gzip.open(target + ".tmp")
475            shutil.copyfileobj(f, open(target, "wb"))
476        f.close()
477        os.remove(target + ".tmp")
478
479    if type(callback) == DownloadProgress:
480        callback.finish()
481
482def localpath_download(domain, filename, **kwargs):
483    """
484    Return local path for the given domain and file. If file does not exist,
485    download it. Additional arguments are passed to the :obj:`download` function.
486    """
487    pathname = localpath(domain, filename)
488    if not os.path.exists(pathname):
489        download(domain, filename, **kwargs)
490    return pathname
491
492def listfiles(domain):
493    """List all files from a domain in a local repository."""
494    dir = localpath(domain)
495    try:
496        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
497    except:
498        files = []
499    okfiles = []
500
501    for file in files:
502        #if file to exists without info
503        if os.path.exists(os.path.join(dir,file[:-5])):
504            #check info format - needs to be valid
505            try:
506                _open_file_info(os.path.join(dir,file))
507                okfiles.append(file[:-5])
508            except:
509                pass
510
511    return okfiles
512
513def remove(domain, filename):
514    """Remove a file from local repository."""
515    filename = localpath(domain, filename)
516    import shutil
517   
518    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
519    todelete = [filename, filename + ".info"] 
520    if "#files" in specialtags:
521        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
522#    print todelete
523    for path in todelete:
524        try:
525            if os.path.isdir(path):
526                shutil.rmtree(path)
527            elif os.path.isfile(path):
528                os.remove(path)
529        except OSError, ex:
530            print "Failed to delete", path, "due to:", ex
531   
532def remove_domain(domain, force=False):
533    """Remove a domain. If force is True, domain is removed even
534    if it is not empty (contains files)."""
535    directory = localpath(domain)
536    if force:
537        import shutil
538        shutil.rmtree(directory)
539    else:
540        os.rmdir(directory)
541
542def listdomains():
543    """List all file domains in the local repository."""
544    dir = localpath()
545    _create_path(dir)
546    files = [ a for a in os.listdir(dir) ]
547    ok = []
548    for file in files:
549        if os.path.isdir(os.path.join(dir, file)):
550            ok.append(file)
551    return ok
552
553def info(domain, filename):
554    """Returns info of a file in a local repository."""
555    target = localpath(domain, filename)
556    return _open_file_info(target + '.info')
557
558def allinfo(domain):
559    """Goes through all files in a domain on a local repository and returns a
560    dictionary, where keys are names of the files and values are their
561    information."""
562    files = listfiles(domain)
563    dic = {}
564    for filename in files:
565        target = localpath(domain, filename)
566        dic[filename] = info(domain, target)
567    return dic
568
569def needs_update(domain, filename, serverfiles=None):
570    """True if a file does not exist in the local repository
571    or if there is a newer version on the server."""
572    if serverfiles == None: serverfiles = ServerFiles()
573    if filename not in listfiles(domain):
574        return True
575    dt_fmt = "%Y-%m-%d %H:%M:%S"
576    dt_local = datetime.datetime.strptime(
577        info(domain, filename)["datetime"][:19], dt_fmt)
578    dt_server = datetime.datetime.strptime(
579        serverfiles.info(domain, filename)["datetime"][:19], dt_fmt)
580    return dt_server > dt_local
581
582def update(domain, filename, serverfiles=None, **kwargs):
583    """Downloads the corresponding file from the server and places it in
584    the local repository, but only if the server copy of the file is newer
585    or the local copy does not exist. An optional  :class:`ServerFiles` object
586    can be passed for authenticated access.
587    """
588    if serverfiles == None: serverfiles = ServerFiles()
589    if needs_update(domain, filename, serverfiles=serverfiles):
590        download(domain, filename, serverfiles=serverfiles, **kwargs)
591       
592def _searchinfo():
593    domains = listdomains()
594    infos = {}
595    for dom in domains:
596        dominfo = allinfo(dom)
597        for a,b in dominfo.items():
598            infos[(dom, a)] = b
599    return infos
600
601def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
602    """
603    sstrings contain a list of search strings
604    """
605    found = []
606
607    for (dom,fn),info in si.items():
608        target = ""
609        if inTag: target += " ".join(info['tags'])
610        if inTitle: target += info['title']
611        if inName: target += fn
612        if not caseSensitive: target = target.lower()
613
614        match = True
615        for s in sstrings:
616            if not caseSensitive:
617                s = s.lower()
618            if s not in target:
619                match= False
620                break
621               
622        if match:
623            found.append((dom,fn))   
624       
625    return found
626
[11396]627
[8042]628def search(sstrings, **kwargs):
629    """Search for files in the local repository where all substrings in a list
630    are contained in at least one chosen field (tag, title, name). Return a
631    list of tuples: first tuple element is the domain of the file, second
632    its name."""
633    si = _searchinfo()
634    return _search(si, sstrings, **kwargs)
635
[11396]636
637def sizeformat(size):
638    """
639    >>> sizeformat(256)
640    256 bytes
641    >>> sizeformat(1024)
642    1.0 KB
643    >>> sizeformat(1.5 * 2 ** 20)
644    1.5 MB
645
646    """
647    for unit in ['bytes', 'KB', 'MB', 'GB', 'TB']:
648        if size < 1024.0:
649            if unit == "bytes":
650                return "%1.0f %s" % (size, unit)
651            else:
652                return "%3.1f %s" % (size, unit)
653        size /= 1024.0
654    return "%.1f PB" % size
655
656
[8042]657class DownloadProgress(ConsoleProgressBar):
658    redirect = None
659    lock = threading.RLock()
[11396]660
[8042]661    def __init__(self, filename, size):
662        print "Downloading", filename
663        ConsoleProgressBar.__init__(self, "progress:", 20)
664        self.size = size
665        self.starttime = time.time()
666        self.speed = 0.0
667
668    def sizeof_fmt(self, num):
[11396]669        return sizeformat(num)
[8042]670
671    def getstring(self):
[11396]672        elapsed = max(time.time() - self.starttime, 0.1)
[11438]673        speed = max(int(self.state * self.size / 100.0 / elapsed), 1)
[8042]674        eta = (100 - self.state) * self.size / 100.0 / speed
[11396]675        return ConsoleProgressBar.getstring(self) + \
676               %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size),
677                                               self.sizeof_fmt(speed),
678                                               eta / 60, eta % 60)
679
[8042]680    def __call__(self, *args, **kwargs):
681        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
682        if self.redirect:
683            self.redirect(self.state)
684        return ret
[11396]685
[8042]686    class RedirectContext(object):
687        def __enter__(self):
688            DownloadProgress.lock.acquire()
689            return DownloadProgress
[11396]690
[8042]691        def __exit__(self, ex_type, value, tb):
692            DownloadProgress.redirect = None
693            DownloadProgress.lock.release()
694            return False
[11396]695
[8042]696    @classmethod
697    def setredirect(cls, redirect):
698        cls.redirect = staticmethod(redirect)
699        return cls.RedirectContext()
[11396]700
[8042]701    @classmethod
702    def __enter__(cls):
703        cls.lock.acquire()
704        return cls
[11396]705
[8042]706    @classmethod
707    def __exit__(cls, exc_type, exc_value, traceback):
708        cls.lock.release()
709        return False
710
[11396]711
[8042]712def consoleupdate(domains=None, searchstr="essential"):
713    domains = domains or listdomains()
714    sf = ServerFiles()
715    info = dict((d, sf.allinfo(d)) for d in domains)
716    def searchmenu():
717        def printmenu():
718            print "\tSearch tags:", search
719            print "\t1. Add tag."
720            print "\t2. Clear tags."
721            print "\t0. Return to main menu."
722            return raw_input("\tSelect option:")
723        search = searchstr
724        while True:
725            response = printmenu().strip()
726            if response == "1":
727                search += " " + raw_input("\tType new tag/tags:")
728            elif response == "2":
729                search = ""
730            elif response == "0":
731                break
732            else:
733                print "\tUnknown option!"
734        return search
735
736    def filemenu(searchstr=""):
737        files = [None]
738        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
739            print "\t%i." % (i + 1), info[dom][file]["title"]
740            files.append((dom, file))
741        print "\t0. Return to main menu."
742        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
743        while True:
744            response = raw_input("\tAction:").strip()
745            if response == "0":
746                break
747            try:
748                action, num = response.split(None, 1)
749                num = int(num)
750            except Exception, ex:
751                print "Unknown option!"
752                continue
753            try:
754                if action.lower() == "d":
755                    download(*(files[num]))
756                    print "\tSuccsessfully downloaded", files[num][-1]
757            except Exception, ex:
758                print "Error occured!", ex
759
760    def printmenu():
761        print "Update database main menu:"
762        print "1. Enter search tags (refine search)."
763        print "2. Print matching available files."
764        print "3. Print all available files."
765        print "4. Update all local files."
766        print "0. Exit."
767        return raw_input("Select option:")
768   
769    while True:
770        try:
771            response = printmenu().strip()
772            if response == "1":
773                searchstr = searchmenu()
774            elif response == "2":
775                filemenu(searchstr)
776            elif response == "3":
777                filemenu("")
778            elif response == "4":
779                update_local_files()
780            elif response == "0":
781                break
782            else:
783                print "Unknown option!"
784        except Exception, ex:
785            print "Error occured:", ex
786
787def update_local_files(verbose=True):
788    sf = ServerFiles()
789    for domain, filename in search(""):
790        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
791        if not uptodate:
792            download(domain, filename, sf)
793        if verbose:
794            print filename, "Ok" if uptodate else "Updated"
795
796def update_by_tags(tags=["essential"], domains=[], verbose=True):
797    sf = ServerFiles()
798    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
799        if domains and domain not in domain:
800            continue
801        if os.path.exists(localpath(domain, filename)+".info"):
802            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
803        else:
804            uptodate = False
805        if not uptodate:
806            download(domain, filename, sf)
807        if verbose:
808            print filename, "Ok" if uptodate else "Updated"
809           
810def _example(myusername, mypassword):
811
812    locallist = listfiles('test')
813    for l in locallist:
814        print info('test', l)
815
816    s = ServerFiles()
817
818    print "testing connection - public"
819    print "AN", s.index()
820
821    #login as an authenticated user
822    s = ServerFiles(username=myusername, password=mypassword)
823   
824    """
825    print "Server search 1"
826    import time
827    t = time.time()
828    print s.search(["rat"])
829    print time.time() - t
830
831    t = time.time()
832    print s.search(["human", "ke"])
833    print time.time() - t
834    """
835
836    print "testing connection - private"
837    print "AN", s.index()
838
839    #create domain
840    try: 
841        s.create_domain("test") 
842    except:
843        print "Failed to create the domain"
844        pass
845
846    files = s.listfiles('test')
847    print "Files in test", files
848
849    print "uploading"
850
851    #upload this file - save it by a different name
852    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
853    #make it public
854    s.unprotect('test', 'osf-test.py')
855
856    #login anonymously
857    s = ServerFiles()
858
859    #list files in the domain "test"
860    files = s.listfiles('test')
861    print "ALL FILES:", files
862
863    for f in files:
864        fi = s.info('test', f) 
865        print "--------------------------------------", f
866        print "INFO", fi
867        print s.downloadFH('test', f).read()[:100] #show first 100 characters
868        print "--------------------------------------"
869
870    #login as an authenticated user
871    s = ServerFiles(username=myusername, password=mypassword)
872
873    print s.listdomains()
874
875    s.remove('test', 'osf-test.py')
876
877    s = ServerFiles()
878
879    print s.listdomains()
880
881
882if __name__ == '__main__':
883    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.