source: orange/Orange/utils/serverfiles.py @ 11653:705107ae06c1

Revision 11653:705107ae06c1, 28.8 KB checked in by Ales Erjavec <ales.erjavec@…>, 8 months ago (diff)

Removed the use of global 'socket.setdefaulttimeout'.

(fixes #1310).

RevLine 
[8042]1"""
2==============================
3Server files (``serverfiles``)
4==============================
5
6.. index:: server files
7
8Server files allows users to download files from a common
9repository residing on the Orange server. It was designed to simplify
10the download and updates of external data sources for Orange Genomics add-on.
11Furthermore, an authenticated user can also manage the repository files with
12this module.
13
14Orange server file repository was created to store large files that do not
15come with Orange installation, but may be required from the user when
16running specific Orange functions. A typical example is Orange Bioinformatics
17package, which relies on large data files storing genome information.
18These do not come pre-installed, but are rather downloaded from the server
19when needed and stored in the local repository. The module provides low-level
20functionality to manage these files, and is used by Orange modules to locate
21the files from the local repository and update/download them when and if needed.
22
23Each managed file is described by domain and the file name.
24Domains are like directories - a place where files are put in.
25
26Domain should consist of less than 255 alphanumeric ASCII characters, whereas
27filenames can be arbitary long and can contain any ASCII character (including
28"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
29domain and filenames. Files can be protected or not. Protected files can
30only be accessed by authenticated users
31
32Local file management
33=====================
34
35The files are saved under Orange's settings directory,
36subdirectory buffer/bigfiles. Each domain is a subdirectory.
37A corresponding info
38file bearing the same name and an extension ".info" is created
39with every download. Info files
40contain title, tags, size and date and time of the file.
41
42.. autofunction:: allinfo
43
44.. autofunction:: download
45
46.. autofunction:: info
47
48.. autofunction:: listdomains
49
50.. autofunction:: listfiles
51
52.. autofunction:: localpath
53
54.. autofunction:: localpath_download
55
56.. autofunction:: needs_update
57
58.. autofunction:: remove
59
60.. autofunction:: remove_domain
61
62.. autofunction:: search
63
64.. autofunction:: update
65
66
67Remote file management
68======================
69
70.. autoclass:: ServerFiles
71    :members:
72
73Examples
74========
75
[9349]76Listing local files, files from the repository and downloading all available files from domain "demo" (:download:`serverfiles1.py <code/serverfiles1.py>`).
[8042]77
78.. literalinclude:: code/serverfiles1.py
79
80A possible output (it depends on the current repository state)::
81
82    My files []
83    Repository files ['orngServerFiles.py', 'urllib2_file.py']
84    Downloading all files in domain 'test'
85    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
86    Downloading orngServerFiles.py
87    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
88    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
89    Downloading urllib2_file.py
90    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
91    My files after download ['urllib2_file.py', 'orngServerFiles.py']
92    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
93
[9349]94A domain with a simple file can be built as follows (:download:`serverfiles2.py <code/serverfiles2.py>`). Of course,
[8042]95the username and password should be valid.
96
97.. literalinclude:: code/serverfiles2.py
98
99A possible output::
100
101    Uploaded.
102    Non-authenticated users see: ['']
103    Authenticated users see: ['titanic.tab']
104    Non-authenticated users now see: ['titanic.tab']
105    orngServerFiles.py file info:
106    {'datetime': '2011-03-15 13:18:53.029000',
107     'size': '45112',
108     'tags': ['basic', 'data set'],
109     'title': 'A sample .tab file'}
110
111"""
112
113import sys
114import socket
115
[11653]116# default socket timeout in seconds
[8042]117timeout = 120
118
[9370]119import urllib
[8042]120import urllib2
121import base64
122
[10580]123from Orange.utils import ConsoleProgressBar
[8042]124import time, threading
125
126import os
127import shutil
128import glob
129import datetime
130import tempfile
131
132#defserver = "localhost:9999/"
133defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
134
135def _parseFileInfo(fir, separ="|||||"):
136    """
137    Parses file info from server.
138    """
139    l= fir.split(separ)
140    fi = {}
141    fi["size"] = l[0]
142    fi["datetime"] = l[1]
143    fi["title"] = l[2]
144    fi["tags"] = l[3].split(";")
145    return fi
146
147def _open_file_info(fname): #no outer usage
148    f = open(fname, 'rt')
149    info = _parseFileInfo(f.read(), separ='\n')
150    f.close()
151    return info
152
153def _save_file_info(fname, info): #no outer usage
154    f = open(fname, 'wt')
155    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
156    f.close()
157
158def _parseList(fl):
159    return fl.split("|||||")
160
161def _parseAllFileInfo(afi):
162    separf = "[[[[["
163    separn = "====="
164    fis = afi.split(separf)
165    out = []
166    for entry in fis:
167        if entry != "":
168            name, info = entry.split(separn)
169            out.append((name, _parseFileInfo(info)))
170
171    return dict(out)
172
173def _create_path_for_file(target):
174    try:
175        os.makedirs(os.path.dirname(target))
176    except OSError:
177        pass
178
179def _create_path(target):
180    try:
181        os.makedirs(target)
182    except OSError:
183        pass
184
185def localpath(domain=None, filename=None):
186    """Return a path for the domain in the local repository. If
187    filename is given, return a path to corresponding file."""
188    import orngEnviron
189    if not domain:
190        return os.path.join(orngEnviron.directoryNames["bufferDir"],
191            "bigfiles")
192    if filename:
193        return os.path.join(orngEnviron.directoryNames["bufferDir"],
194            "bigfiles", domain, filename)
195    else:
196        return os.path.join(orngEnviron.directoryNames["bufferDir"],
197            "bigfiles", domain)
198
199class ServerFiles(object):
200    """
201    To work with the repository, you need to create an instance of
202    ServerFiles object. To access the repository as an authenticated user, a
203    username and password should be passed to the constructor. All password
204    protected operations and transfers are secured by SSL; this secures
205    both password and content.
206
207    Repository files are set as protected when first uploaded: only
208    authenticated users can see them. They need to be unprotected for
209    public use.
210    """
211
212    def __init__(self, username=None, password=None, server=None, access_code=None):
213        """
214        Creates a ServerFiles instance. Pass your username and password
215        to use the repository as an authenticated user. If you want to use
216        your access code (as an non-authenticated user), pass it also.
217        """
218        if not server:
219            server = defserver
220        self.server = server
221        self.secureroot = 'https://' + self.server + 'private/'
222        self.publicroot = 'http://' + self.server + 'public/'
223        self.username = username
224        self.password = password
225        self.access_code = access_code
226        self.searchinfo = None
227
[9370]228    def _getOpener(self, multipart=False):
[8042]229        opener = urllib2.build_opener()
230        return opener
231 
232    def upload(self, domain, filename, file, title="", tags=[]):
233        """ Uploads a file "file" to the domain where it is saved with filename
234        "filename". If file does not exist yet, set it as protected. Parameter
235        file can be a file handle open for reading or a file name.
236        """
237        if isinstance(file, basestring):
238            file = open(file, 'rb')
239
240        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
241        return self._open('upload', data)
242
243    def create_domain(self, domain):
244        """Create a server domain."""
245        return self._open('createdomain', { 'domain': domain })
246
247    def remove_domain(self, domain, force=False):
248        """Remove a domain. If force is True, domain is removed even
249        if it is not empty (contains files)."""
250        data = { 'domain': domain }
251        if force:
252            data['force'] = True
253        return self._open('removedomain', data)
254
255    def remove(self, domain, filename):
256        """Remove a file from the server repository."""
257        return self._open('remove', { 'domain': domain, 'filename': filename })
258
259    def unprotect(self, domain, filename):
260        """Put a file into public use."""
261        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
262
263    def protect(self, domain, filename, access_code="1"):
264        """Hide file from non-authenticated users. If an access code (string)
265        is passed, the file will be available to authenticated users and
266        non-authenticated users with that access code."""
267        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
268
269    def protection(self, domain, filename):
270        """Return file protection. Legend: "0" - public use,
271        "1" - for authenticated users only, anything else
272        represents a specific access code.
273        """
274        return self._open('protection', { 'domain': domain, 'filename': filename })
275   
276    def listfiles(self, domain):
277        """List all files in a repository domain."""
278        return _parseList(self._open('list', { 'domain': domain }))
279
280    def listdomains(self):
281        """List all domains on repository."""
282        return _parseList(self._open('listdomains', {}))
283
284    def downloadFH(self, *args, **kwargs):
285        """Return open file handle of requested file from the server repository given the domain and the filename."""
286        if self._authen(): return self.secdownloadFH(*args, **kwargs)
287        else: return self.pubdownloadFH(*args, **kwargs)
288
289    def download(self, domain, filename, target, callback=None):
290        """
291        Downloads file from the repository to a given target name. Callback
292        can be a function without arguments. It will be called once for each
293        downloaded percent of file: 100 times for the whole file.
294        """
295        _create_path_for_file(target)
296
297        fdown = self.downloadFH(domain, filename)
298        size = int(fdown.headers.getheader('content-length'))
299
300        f = tempfile.TemporaryFile()
301 
302        chunksize = 1024*8
303        lastchunkreport= 0.0001
304
305        readb = 0
[11438]306        # in case size == 0 skip the loop
307        while size > 0:
[8042]308            buf = fdown.read(chunksize)
309            readb += len(buf)
310
[11438]311            while float(readb) / size > lastchunkreport+0.01:
[8042]312                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
313                lastchunkreport += 0.01
314                if callback:
315                    callback()
316            if not buf:
317                break
318            f.write(buf)
319
320        fdown.close()
321        f.seek(0)
322
323        shutil.copyfileobj(f, open(target, "wb"))
324
325        if callback:
326            callback()
327
328    def _searchinfo(self):
329        domains = self.listdomains()
330        infos = {}
331        for dom in domains:
332            dominfo = self.allinfo(dom)
333            for a,b in dominfo.items():
334                infos[(dom, a)] = b
335        return infos
336
337    def search(self, sstrings, **kwargs):
338        """
339        Search for files on the repository where all substrings in a list
340        are contained in at least one choosen field (tag, title, name). Return
341        a list of tuples: first tuple element is the file's domain, second its
342        name. As for now the search is performed locally, therefore
343        information on files in repository is transfered on first call of
344        this function.
345        """
346        if not self.searchinfo:
347            self.searchinfo = self._searchinfo()
348        return _search(self.searchinfo, sstrings, **kwargs)
349
350    def info(self, domain, filename):
351        """Return a dictionary containing repository file info.
352        Keys: title, tags, size, datetime."""
353        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
354
355    def downloadFH(self, domain, filename):
356        """Return a file handle to the file that we would like to download."""
357        return self._handle('download', { 'domain': domain, 'filename': filename })
358
359    def list(self, domain):
360        return _parseList(self._open('list', { 'domain': domain }))
361
362    def listdomains(self):
363        """List all domains on repository."""
364        return _parseList(self._open('listdomains', {}))
365
366    def allinfo(self, domain):
367        """Go through all accessible files in a given domain and return a
368        dictionary, where key is file's name and value its info.
369        """
370        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
371
372    def index(self):
373        return self._open('index', {})
374
375    def _authen(self):
376        """
377        Did the user choose authentication?
378        """
379        if self.username and self.password:
380            return True
381        else:
382            return False
383
384    def _server_request(self, root, command, data, repeat=2):
385        def do():
386            opener = self._getOpener()
[9370]387           
[8042]388            if data:
[9370]389                if command == "upload":
390                    # Need to use poster to handle multipart post
391                    try:
392                        import poster.streaminghttp as psh
393                        import poster.encode
394                    except ImportError:
395                        raise ImportError("You need to install 'poster' (http://pypi.python.org/pypi/poster) to be able to upload files.")
396               
397                    handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
398                    opener = urllib2.build_opener(*handlers)
399                    datagen, headers = poster.encode.multipart_encode(data)
400                    request = urllib2.Request(root+command, datagen, headers)
401                else:
402                    request = urllib2.Request(root+command, urllib.urlencode(data))
[8042]403            else:
404                request = urllib2.Request(root+command)
405
406            #directy add authorization headers
407            if self._authen():
408                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
409                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
[11653]410
411            return opener.open(request, timeout=timeout)
412
[8042]413        if repeat <= 0:
414            return do()
415        else:
416            try:
417                return do()
418            except:
419                return self._server_request(root, command, data, repeat=repeat-1)
420   
421    def _handle(self, command, data):
422        data2 = self._addAccessCode(data)
423        addr = self.publicroot
424        if self._authen():
425            addr = self.secureroot
426        return self._server_request(addr, command, data)
427
428    def _open(self, command, data):
429        return self._handle(command, data).read()
430
431    def _addAccessCode(self, data):
432        if self.access_code != None:
433            data = data.copy()
434            data["access_code"] = self.access_code
435        return data
436
437def download(domain, filename, serverfiles=None, callback=None, 
438    extract=True, verbose=True):
439    """Downloads file from the repository to local orange installation.
440    To download files as an authenticated user you should also pass an
441    instance of ServerFiles class. Callback can be a function without
442    arguments. It will be called once for each downloaded percent of
443    file: 100 times for the whole file."""
444
445    if not serverfiles:
446        serverfiles = ServerFiles()
447
448    info = serverfiles.info(domain, filename)
449    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
450    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
451    target = localpath(domain, filename)
452    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
453    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
454   
455    #file saved, now save info file
456
457    _save_file_info(target + '.info', info)
458   
459    if extract:
460        import tarfile, gzip, shutil
461        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
462            f = tarfile.open(target + ".tmp")
463            f.extractall(localpath(domain))
464            shutil.copyfile(target + ".tmp", target)
[8997]465        elif filename.endswith(".tar.gz"):
[8042]466            f = tarfile.open(target + ".tmp")
467            try:
468                os.mkdir(target)
469            except Exception:
470                pass
471            f.extractall(target)
472        elif specialtags.get("#compression") == "gz":
473            f = gzip.open(target + ".tmp")
474            shutil.copyfileobj(f, open(target, "wb"))
475        f.close()
476        os.remove(target + ".tmp")
477
478    if type(callback) == DownloadProgress:
479        callback.finish()
480
481def localpath_download(domain, filename, **kwargs):
482    """
483    Return local path for the given domain and file. If file does not exist,
484    download it. Additional arguments are passed to the :obj:`download` function.
485    """
486    pathname = localpath(domain, filename)
487    if not os.path.exists(pathname):
488        download(domain, filename, **kwargs)
489    return pathname
490
491def listfiles(domain):
492    """List all files from a domain in a local repository."""
493    dir = localpath(domain)
494    try:
495        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
496    except:
497        files = []
498    okfiles = []
499
500    for file in files:
501        #if file to exists without info
502        if os.path.exists(os.path.join(dir,file[:-5])):
503            #check info format - needs to be valid
504            try:
505                _open_file_info(os.path.join(dir,file))
506                okfiles.append(file[:-5])
507            except:
508                pass
509
510    return okfiles
511
512def remove(domain, filename):
513    """Remove a file from local repository."""
514    filename = localpath(domain, filename)
515    import shutil
516   
517    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
518    todelete = [filename, filename + ".info"] 
519    if "#files" in specialtags:
520        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
521#    print todelete
522    for path in todelete:
523        try:
524            if os.path.isdir(path):
525                shutil.rmtree(path)
526            elif os.path.isfile(path):
527                os.remove(path)
528        except OSError, ex:
529            print "Failed to delete", path, "due to:", ex
530   
531def remove_domain(domain, force=False):
532    """Remove a domain. If force is True, domain is removed even
533    if it is not empty (contains files)."""
534    directory = localpath(domain)
535    if force:
536        import shutil
537        shutil.rmtree(directory)
538    else:
539        os.rmdir(directory)
540
541def listdomains():
542    """List all file domains in the local repository."""
543    dir = localpath()
544    _create_path(dir)
545    files = [ a for a in os.listdir(dir) ]
546    ok = []
547    for file in files:
548        if os.path.isdir(os.path.join(dir, file)):
549            ok.append(file)
550    return ok
551
552def info(domain, filename):
553    """Returns info of a file in a local repository."""
554    target = localpath(domain, filename)
555    return _open_file_info(target + '.info')
556
557def allinfo(domain):
558    """Goes through all files in a domain on a local repository and returns a
559    dictionary, where keys are names of the files and values are their
560    information."""
561    files = listfiles(domain)
562    dic = {}
563    for filename in files:
564        target = localpath(domain, filename)
565        dic[filename] = info(domain, target)
566    return dic
567
568def needs_update(domain, filename, serverfiles=None):
569    """True if a file does not exist in the local repository
570    or if there is a newer version on the server."""
571    if serverfiles == None: serverfiles = ServerFiles()
572    if filename not in listfiles(domain):
573        return True
574    dt_fmt = "%Y-%m-%d %H:%M:%S"
575    dt_local = datetime.datetime.strptime(
576        info(domain, filename)["datetime"][:19], dt_fmt)
577    dt_server = datetime.datetime.strptime(
578        serverfiles.info(domain, filename)["datetime"][:19], dt_fmt)
579    return dt_server > dt_local
580
581def update(domain, filename, serverfiles=None, **kwargs):
582    """Downloads the corresponding file from the server and places it in
583    the local repository, but only if the server copy of the file is newer
584    or the local copy does not exist. An optional  :class:`ServerFiles` object
585    can be passed for authenticated access.
586    """
587    if serverfiles == None: serverfiles = ServerFiles()
588    if needs_update(domain, filename, serverfiles=serverfiles):
589        download(domain, filename, serverfiles=serverfiles, **kwargs)
590       
591def _searchinfo():
592    domains = listdomains()
593    infos = {}
594    for dom in domains:
595        dominfo = allinfo(dom)
596        for a,b in dominfo.items():
597            infos[(dom, a)] = b
598    return infos
599
600def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
601    """
602    sstrings contain a list of search strings
603    """
604    found = []
605
606    for (dom,fn),info in si.items():
607        target = ""
608        if inTag: target += " ".join(info['tags'])
609        if inTitle: target += info['title']
610        if inName: target += fn
611        if not caseSensitive: target = target.lower()
612
613        match = True
614        for s in sstrings:
615            if not caseSensitive:
616                s = s.lower()
617            if s not in target:
618                match= False
619                break
620               
621        if match:
622            found.append((dom,fn))   
623       
624    return found
625
[11396]626
[8042]627def search(sstrings, **kwargs):
628    """Search for files in the local repository where all substrings in a list
629    are contained in at least one chosen field (tag, title, name). Return a
630    list of tuples: first tuple element is the domain of the file, second
631    its name."""
632    si = _searchinfo()
633    return _search(si, sstrings, **kwargs)
634
[11396]635
636def sizeformat(size):
637    """
638    >>> sizeformat(256)
639    256 bytes
640    >>> sizeformat(1024)
641    1.0 KB
642    >>> sizeformat(1.5 * 2 ** 20)
643    1.5 MB
644
645    """
646    for unit in ['bytes', 'KB', 'MB', 'GB', 'TB']:
647        if size < 1024.0:
648            if unit == "bytes":
649                return "%1.0f %s" % (size, unit)
650            else:
651                return "%3.1f %s" % (size, unit)
652        size /= 1024.0
653    return "%.1f PB" % size
654
655
[8042]656class DownloadProgress(ConsoleProgressBar):
657    redirect = None
658    lock = threading.RLock()
[11396]659
[8042]660    def __init__(self, filename, size):
661        print "Downloading", filename
662        ConsoleProgressBar.__init__(self, "progress:", 20)
663        self.size = size
664        self.starttime = time.time()
665        self.speed = 0.0
666
667    def sizeof_fmt(self, num):
[11396]668        return sizeformat(num)
[8042]669
670    def getstring(self):
[11396]671        elapsed = max(time.time() - self.starttime, 0.1)
[11438]672        speed = max(int(self.state * self.size / 100.0 / elapsed), 1)
[8042]673        eta = (100 - self.state) * self.size / 100.0 / speed
[11396]674        return ConsoleProgressBar.getstring(self) + \
675               %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size),
676                                               self.sizeof_fmt(speed),
677                                               eta / 60, eta % 60)
678
[8042]679    def __call__(self, *args, **kwargs):
680        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
681        if self.redirect:
682            self.redirect(self.state)
683        return ret
[11396]684
[8042]685    class RedirectContext(object):
686        def __enter__(self):
687            DownloadProgress.lock.acquire()
688            return DownloadProgress
[11396]689
[8042]690        def __exit__(self, ex_type, value, tb):
691            DownloadProgress.redirect = None
692            DownloadProgress.lock.release()
693            return False
[11396]694
[8042]695    @classmethod
696    def setredirect(cls, redirect):
697        cls.redirect = staticmethod(redirect)
698        return cls.RedirectContext()
[11396]699
[8042]700    @classmethod
701    def __enter__(cls):
702        cls.lock.acquire()
703        return cls
[11396]704
[8042]705    @classmethod
706    def __exit__(cls, exc_type, exc_value, traceback):
707        cls.lock.release()
708        return False
709
[11396]710
[8042]711def consoleupdate(domains=None, searchstr="essential"):
712    domains = domains or listdomains()
713    sf = ServerFiles()
714    info = dict((d, sf.allinfo(d)) for d in domains)
715    def searchmenu():
716        def printmenu():
717            print "\tSearch tags:", search
718            print "\t1. Add tag."
719            print "\t2. Clear tags."
720            print "\t0. Return to main menu."
721            return raw_input("\tSelect option:")
722        search = searchstr
723        while True:
724            response = printmenu().strip()
725            if response == "1":
726                search += " " + raw_input("\tType new tag/tags:")
727            elif response == "2":
728                search = ""
729            elif response == "0":
730                break
731            else:
732                print "\tUnknown option!"
733        return search
734
735    def filemenu(searchstr=""):
736        files = [None]
737        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
738            print "\t%i." % (i + 1), info[dom][file]["title"]
739            files.append((dom, file))
740        print "\t0. Return to main menu."
741        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
742        while True:
743            response = raw_input("\tAction:").strip()
744            if response == "0":
745                break
746            try:
747                action, num = response.split(None, 1)
748                num = int(num)
749            except Exception, ex:
750                print "Unknown option!"
751                continue
752            try:
753                if action.lower() == "d":
754                    download(*(files[num]))
755                    print "\tSuccsessfully downloaded", files[num][-1]
756            except Exception, ex:
757                print "Error occured!", ex
758
759    def printmenu():
760        print "Update database main menu:"
761        print "1. Enter search tags (refine search)."
762        print "2. Print matching available files."
763        print "3. Print all available files."
764        print "4. Update all local files."
765        print "0. Exit."
766        return raw_input("Select option:")
767   
768    while True:
769        try:
770            response = printmenu().strip()
771            if response == "1":
772                searchstr = searchmenu()
773            elif response == "2":
774                filemenu(searchstr)
775            elif response == "3":
776                filemenu("")
777            elif response == "4":
778                update_local_files()
779            elif response == "0":
780                break
781            else:
782                print "Unknown option!"
783        except Exception, ex:
784            print "Error occured:", ex
785
786def update_local_files(verbose=True):
787    sf = ServerFiles()
788    for domain, filename in search(""):
789        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
790        if not uptodate:
791            download(domain, filename, sf)
792        if verbose:
793            print filename, "Ok" if uptodate else "Updated"
794
795def update_by_tags(tags=["essential"], domains=[], verbose=True):
796    sf = ServerFiles()
797    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
798        if domains and domain not in domain:
799            continue
800        if os.path.exists(localpath(domain, filename)+".info"):
801            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
802        else:
803            uptodate = False
804        if not uptodate:
805            download(domain, filename, sf)
806        if verbose:
807            print filename, "Ok" if uptodate else "Updated"
808           
809def _example(myusername, mypassword):
810
811    locallist = listfiles('test')
812    for l in locallist:
813        print info('test', l)
814
815    s = ServerFiles()
816
817    print "testing connection - public"
818    print "AN", s.index()
819
820    #login as an authenticated user
821    s = ServerFiles(username=myusername, password=mypassword)
822   
823    """
824    print "Server search 1"
825    import time
826    t = time.time()
827    print s.search(["rat"])
828    print time.time() - t
829
830    t = time.time()
831    print s.search(["human", "ke"])
832    print time.time() - t
833    """
834
835    print "testing connection - private"
836    print "AN", s.index()
837
838    #create domain
839    try: 
840        s.create_domain("test") 
841    except:
842        print "Failed to create the domain"
843        pass
844
845    files = s.listfiles('test')
846    print "Files in test", files
847
848    print "uploading"
849
850    #upload this file - save it by a different name
851    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
852    #make it public
853    s.unprotect('test', 'osf-test.py')
854
855    #login anonymously
856    s = ServerFiles()
857
858    #list files in the domain "test"
859    files = s.listfiles('test')
860    print "ALL FILES:", files
861
862    for f in files:
863        fi = s.info('test', f) 
864        print "--------------------------------------", f
865        print "INFO", fi
866        print s.downloadFH('test', f).read()[:100] #show first 100 characters
867        print "--------------------------------------"
868
869    #login as an authenticated user
870    s = ServerFiles(username=myusername, password=mypassword)
871
872    print s.listdomains()
873
874    s.remove('test', 'osf-test.py')
875
876    s = ServerFiles()
877
878    print s.listdomains()
879
880
881if __name__ == '__main__':
882    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.