source: orange/Orange/utils/serverfiles.py @ 11777:77ff33109990

Revision 11777:77ff33109990, 30.9 KB checked in by Ales Erjavec <ales.erjavec@…>, 5 months ago (diff)

Added locking for thread safe downloads.

Line 
1"""
2==============================
3Server files (``serverfiles``)
4==============================
5
6.. index:: server files
7
8Server files allows users to download files from a common
9repository residing on the Orange server. It was designed to simplify
10the download and updates of external data sources for Orange Genomics add-on.
11Furthermore, an authenticated user can also manage the repository files with
12this module.
13
14Orange server file repository was created to store large files that do not
15come with Orange installation, but may be required from the user when
16running specific Orange functions. A typical example is Orange Bioinformatics
17package, which relies on large data files storing genome information.
18These do not come pre-installed, but are rather downloaded from the server
19when needed and stored in the local repository. The module provides low-level
20functionality to manage these files, and is used by Orange modules to locate
21the files from the local repository and update/download them when and if needed.
22
23Each managed file is described by domain and the file name.
24Domains are like directories - a place where files are put in.
25
26Domain should consist of less than 255 alphanumeric ASCII characters, whereas
27filenames can be arbitary long and can contain any ASCII character (including
28"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
29domain and filenames. Files can be protected or not. Protected files can
30only be accessed by authenticated users
31
32Local file management
33=====================
34
35The files are saved under Orange's settings directory,
36subdirectory buffer/bigfiles. Each domain is a subdirectory.
37A corresponding info
38file bearing the same name and an extension ".info" is created
39with every download. Info files
40contain title, tags, size and date and time of the file.
41
42.. autofunction:: allinfo
43
44.. autofunction:: download
45
46.. autofunction:: info
47
48.. autofunction:: listdomains
49
50.. autofunction:: listfiles
51
52.. autofunction:: localpath
53
54.. autofunction:: localpath_download
55
56.. autofunction:: needs_update
57
58.. autofunction:: remove
59
60.. autofunction:: remove_domain
61
62.. autofunction:: search
63
64.. autofunction:: update
65
66
67Remote file management
68======================
69
70.. autoclass:: ServerFiles
71    :members:
72
73Examples
74========
75
76Listing local files, files from the repository and downloading all available files from domain "demo" (:download:`serverfiles1.py <code/serverfiles1.py>`).
77
78.. literalinclude:: code/serverfiles1.py
79
80A possible output (it depends on the current repository state)::
81
82    My files []
83    Repository files ['orngServerFiles.py', 'urllib2_file.py']
84    Downloading all files in domain 'test'
85    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
86    Downloading orngServerFiles.py
87    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
88    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
89    Downloading urllib2_file.py
90    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
91    My files after download ['urllib2_file.py', 'orngServerFiles.py']
92    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
93
94A domain with a simple file can be built as follows (:download:`serverfiles2.py <code/serverfiles2.py>`). Of course,
95the username and password should be valid.
96
97.. literalinclude:: code/serverfiles2.py
98
99A possible output::
100
101    Uploaded.
102    Non-authenticated users see: ['']
103    Authenticated users see: ['titanic.tab']
104    Non-authenticated users now see: ['titanic.tab']
105    orngServerFiles.py file info:
106    {'datetime': '2011-03-15 13:18:53.029000',
107     'size': '45112',
108     'tags': ['basic', 'data set'],
109     'title': 'A sample .tab file'}
110
111"""
112
113import sys
114import socket
115
116# default socket timeout in seconds
117timeout = 120
118
119import urllib
120import urllib2
121import base64
122import functools
123
124from contextlib import contextmanager
125
126from Orange.utils import ConsoleProgressBar
127import time, threading
128
129import os
130import shutil
131import glob
132import datetime
133import tempfile
134
135#defserver = "localhost:9999/"
136defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
137
138def _parseFileInfo(fir, separ="|||||"):
139    """
140    Parses file info from server.
141    """
142    l= fir.split(separ)
143    fi = {}
144    fi["size"] = l[0]
145    fi["datetime"] = l[1]
146    fi["title"] = l[2]
147    fi["tags"] = l[3].split(";")
148    return fi
149
150def _open_file_info(fname): #no outer usage
151    f = open(fname, 'rt')
152    info = _parseFileInfo(f.read(), separ='\n')
153    f.close()
154    return info
155
156def _save_file_info(fname, info): #no outer usage
157    f = open(fname, 'wt')
158    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
159    f.close()
160
161def _parseList(fl):
162    return fl.split("|||||")
163
164def _parseAllFileInfo(afi):
165    separf = "[[[[["
166    separn = "====="
167    fis = afi.split(separf)
168    out = []
169    for entry in fis:
170        if entry != "":
171            name, info = entry.split(separn)
172            out.append((name, _parseFileInfo(info)))
173
174    return dict(out)
175
176def _create_path_for_file(target):
177    try:
178        os.makedirs(os.path.dirname(target))
179    except OSError:
180        pass
181
182def _create_path(target):
183    try:
184        os.makedirs(target)
185    except OSError:
186        pass
187
188def localpath(domain=None, filename=None):
189    """Return a path for the domain in the local repository. If
190    filename is given, return a path to corresponding file."""
191    import orngEnviron
192    if not domain:
193        return os.path.join(orngEnviron.directoryNames["bufferDir"],
194            "bigfiles")
195    if filename:
196        return os.path.join(orngEnviron.directoryNames["bufferDir"],
197            "bigfiles", domain, filename)
198    else:
199        return os.path.join(orngEnviron.directoryNames["bufferDir"],
200            "bigfiles", domain)
201
202class ServerFiles(object):
203    """
204    To work with the repository, you need to create an instance of
205    ServerFiles object. To access the repository as an authenticated user, a
206    username and password should be passed to the constructor. All password
207    protected operations and transfers are secured by SSL; this secures
208    both password and content.
209
210    Repository files are set as protected when first uploaded: only
211    authenticated users can see them. They need to be unprotected for
212    public use.
213    """
214
215    def __init__(self, username=None, password=None, server=None, access_code=None):
216        """
217        Creates a ServerFiles instance. Pass your username and password
218        to use the repository as an authenticated user. If you want to use
219        your access code (as an non-authenticated user), pass it also.
220        """
221        if not server:
222            server = defserver
223        self.server = server
224        self.secureroot = 'https://' + self.server + 'private/'
225        self.publicroot = 'http://' + self.server + 'public/'
226        self.username = username
227        self.password = password
228        self.access_code = access_code
229        self.searchinfo = None
230
231    def _getOpener(self, multipart=False):
232        opener = urllib2.build_opener()
233        return opener
234 
235    def upload(self, domain, filename, file, title="", tags=[]):
236        """ Uploads a file "file" to the domain where it is saved with filename
237        "filename". If file does not exist yet, set it as protected. Parameter
238        file can be a file handle open for reading or a file name.
239        """
240        if isinstance(file, basestring):
241            file = open(file, 'rb')
242
243        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
244        return self._open('upload', data)
245
246    def create_domain(self, domain):
247        """Create a server domain."""
248        return self._open('createdomain', { 'domain': domain })
249
250    def remove_domain(self, domain, force=False):
251        """Remove a domain. If force is True, domain is removed even
252        if it is not empty (contains files)."""
253        data = { 'domain': domain }
254        if force:
255            data['force'] = True
256        return self._open('removedomain', data)
257
258    def remove(self, domain, filename):
259        """Remove a file from the server repository."""
260        return self._open('remove', { 'domain': domain, 'filename': filename })
261
262    def unprotect(self, domain, filename):
263        """Put a file into public use."""
264        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
265
266    def protect(self, domain, filename, access_code="1"):
267        """Hide file from non-authenticated users. If an access code (string)
268        is passed, the file will be available to authenticated users and
269        non-authenticated users with that access code."""
270        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
271
272    def protection(self, domain, filename):
273        """Return file protection. Legend: "0" - public use,
274        "1" - for authenticated users only, anything else
275        represents a specific access code.
276        """
277        return self._open('protection', { 'domain': domain, 'filename': filename })
278   
279    def listfiles(self, domain):
280        """List all files in a repository domain."""
281        return _parseList(self._open('list', { 'domain': domain }))
282
283    def listdomains(self):
284        """List all domains on repository."""
285        return _parseList(self._open('listdomains', {}))
286
287    def downloadFH(self, *args, **kwargs):
288        """Return open file handle of requested file from the server repository given the domain and the filename."""
289        if self._authen(): return self.secdownloadFH(*args, **kwargs)
290        else: return self.pubdownloadFH(*args, **kwargs)
291
292    def download(self, domain, filename, target, callback=None):
293        """
294        Downloads file from the repository to a given target name. Callback
295        can be a function without arguments. It will be called once for each
296        downloaded percent of file: 100 times for the whole file.
297        """
298        _create_path_for_file(target)
299
300        fdown = self.downloadFH(domain, filename)
301        size = int(fdown.headers.getheader('content-length'))
302
303        f = tempfile.TemporaryFile()
304 
305        chunksize = 1024*8
306        lastchunkreport= 0.0001
307
308        readb = 0
309        # in case size == 0 skip the loop
310        while size > 0:
311            buf = fdown.read(chunksize)
312            readb += len(buf)
313
314            while float(readb) / size > lastchunkreport+0.01:
315                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
316                lastchunkreport += 0.01
317                if callback:
318                    callback()
319            if not buf:
320                break
321            f.write(buf)
322
323        fdown.close()
324        f.seek(0)
325
326        shutil.copyfileobj(f, open(target, "wb"))
327
328        if callback:
329            callback()
330
331    def _searchinfo(self):
332        domains = self.listdomains()
333        infos = {}
334        for dom in domains:
335            dominfo = self.allinfo(dom)
336            for a,b in dominfo.items():
337                infos[(dom, a)] = b
338        return infos
339
340    def search(self, sstrings, **kwargs):
341        """
342        Search for files on the repository where all substrings in a list
343        are contained in at least one choosen field (tag, title, name). Return
344        a list of tuples: first tuple element is the file's domain, second its
345        name. As for now the search is performed locally, therefore
346        information on files in repository is transfered on first call of
347        this function.
348        """
349        if not self.searchinfo:
350            self.searchinfo = self._searchinfo()
351        return _search(self.searchinfo, sstrings, **kwargs)
352
353    def info(self, domain, filename):
354        """Return a dictionary containing repository file info.
355        Keys: title, tags, size, datetime."""
356        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
357
358    def downloadFH(self, domain, filename):
359        """Return a file handle to the file that we would like to download."""
360        return self._handle('download', { 'domain': domain, 'filename': filename })
361
362    def list(self, domain):
363        return _parseList(self._open('list', { 'domain': domain }))
364
365    def listdomains(self):
366        """List all domains on repository."""
367        return _parseList(self._open('listdomains', {}))
368
369    def allinfo(self, domain):
370        """Go through all accessible files in a given domain and return a
371        dictionary, where key is file's name and value its info.
372        """
373        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
374
375    def index(self):
376        return self._open('index', {})
377
378    def _authen(self):
379        """
380        Did the user choose authentication?
381        """
382        if self.username and self.password:
383            return True
384        else:
385            return False
386
387    def _server_request(self, root, command, data, repeat=2):
388        def do():
389            opener = self._getOpener()
390           
391            if data:
392                if command == "upload":
393                    # Need to use poster to handle multipart post
394                    try:
395                        import poster.streaminghttp as psh
396                        import poster.encode
397                    except ImportError:
398                        raise ImportError("You need to install 'poster' (http://pypi.python.org/pypi/poster) to be able to upload files.")
399               
400                    handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
401                    opener = urllib2.build_opener(*handlers)
402                    datagen, headers = poster.encode.multipart_encode(data)
403                    request = urllib2.Request(root+command, datagen, headers)
404                else:
405                    request = urllib2.Request(root+command, urllib.urlencode(data))
406            else:
407                request = urllib2.Request(root+command)
408
409            #directy add authorization headers
410            if self._authen():
411                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
412                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
413
414            return opener.open(request, timeout=timeout)
415
416        if repeat <= 0:
417            return do()
418        else:
419            try:
420                return do()
421            except:
422                return self._server_request(root, command, data, repeat=repeat-1)
423   
424    def _handle(self, command, data):
425        data2 = self._addAccessCode(data)
426        addr = self.publicroot
427        if self._authen():
428            addr = self.secureroot
429        return self._server_request(addr, command, data)
430
431    def _open(self, command, data):
432        return self._handle(command, data).read()
433
434    def _addAccessCode(self, data):
435        if self.access_code != None:
436            data = data.copy()
437            data["access_code"] = self.access_code
438        return data
439
440
441def _keyed_lock(lock_constructor=threading.Lock):
442    lock = threading.Lock()
443    locks = {}
444
445    def get_lock(key):
446        with lock:
447            if key not in locks:
448                locks[key] = lock_constructor()
449
450            return locks[key]
451    return get_lock
452
453
454class _Lock(object):
455    """
456    Like :class:`threading.Lock` but raises an error on reentrant acquire.
457    """
458    def __init__(self):
459        self.__lock = threading.RLock()
460        self.__locking_thread = None
461
462    def acquire(self, blocking=True):
463        if self.__lock.acquire(blocking):
464            if self.__locking_thread is None:
465                self.__locking_thread = threading.current_thread()
466            else:
467                assert self.__locking_thread == threading.current_thread()
468                self.__lock.release()
469                raise Exception("Recursive lock acquire!")
470
471            return True
472        else:
473            return False
474
475    def release(self):
476        assert self.__locking_thread is not None
477        self.__locking_thread = None
478        self.__lock.release()
479
480    def locked(self):
481        return self.__lock.locked()
482
483    def __enter__(self):
484        self.acquire()
485
486    def __exit__(self, *arg):
487        self.release()
488
489# _get_lock = _keyed_lock(threading.RLock)
490_get_lock = _keyed_lock(_Lock)
491
492
493@contextmanager
494def _lock_file(domain, filename, blocking=False):
495    path = localpath(domain, filename)
496    path = os.path.normpath(os.path.realpath(path))
497#     log.debug("locking: %s", path)
498    lock = _get_lock(path)
499    if lock.acquire(blocking):
500#         log.debug("got lock on: %s", path)
501        try:
502            yield
503        finally:
504            lock.release()
505#             log.debug("Released lock on: %s",  path)
506    else:
507        raise Exception("Could not acquire lock")
508
509
510def _locked(f):
511    @functools.wraps(f)
512    def func(domain, filename, *args, **kwargs):
513        with _lock_file(domain, filename, blocking=True):
514            return f(domain, filename, *args, **kwargs)
515    func.unwraped = f
516    return func
517
518
519@_locked
520def download(domain, filename, serverfiles=None, callback=None,
521             extract=True, verbose=True):
522    """Downloads file from the repository to local orange installation.
523    To download files as an authenticated user you should also pass an
524    instance of ServerFiles class. Callback can be a function without
525    arguments. It will be called once for each downloaded percent of
526    file: 100 times for the whole file."""
527
528    if not serverfiles:
529        serverfiles = ServerFiles()
530
531    info = serverfiles.info(domain, filename)
532    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
533    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
534    target = localpath(domain, filename)
535    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
536    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
537   
538    #file saved, now save info file
539
540    _save_file_info(target + '.info', info)
541   
542    if extract:
543        import tarfile, gzip, shutil
544        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
545            f = tarfile.open(target + ".tmp")
546            f.extractall(localpath(domain))
547            shutil.copyfile(target + ".tmp", target)
548        elif filename.endswith(".tar.gz"):
549            f = tarfile.open(target + ".tmp")
550            try:
551                os.mkdir(target)
552            except Exception:
553                pass
554            f.extractall(target)
555        elif specialtags.get("#compression") == "gz":
556            f = gzip.open(target + ".tmp")
557            shutil.copyfileobj(f, open(target, "wb"))
558        f.close()
559        os.remove(target + ".tmp")
560
561    if type(callback) == DownloadProgress:
562        callback.finish()
563
564
565@_locked
566def localpath_download(domain, filename, **kwargs):
567    """
568    Return local path for the given domain and file. If file does not exist,
569    download it. Additional arguments are passed to the :obj:`download` function.
570    """
571    pathname = localpath(domain, filename)
572    if not os.path.exists(pathname):
573        download.unwraped(domain, filename, **kwargs)
574    return pathname
575
576def listfiles(domain):
577    """List all files from a domain in a local repository."""
578    dir = localpath(domain)
579    try:
580        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
581    except:
582        files = []
583    okfiles = []
584
585    for file in files:
586        #if file to exists without info
587        if os.path.exists(os.path.join(dir,file[:-5])):
588            #check info format - needs to be valid
589            try:
590                _open_file_info(os.path.join(dir,file))
591                okfiles.append(file[:-5])
592            except:
593                pass
594
595    return okfiles
596
597
598@_locked
599def remove(domain, filename):
600    """Remove a file from local repository."""
601    filename = localpath(domain, filename)
602    import shutil
603   
604    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
605    todelete = [filename, filename + ".info"] 
606    if "#files" in specialtags:
607        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
608#    print todelete
609    for path in todelete:
610        try:
611            if os.path.isdir(path):
612                shutil.rmtree(path)
613            elif os.path.isfile(path):
614                os.remove(path)
615        except OSError, ex:
616            print "Failed to delete", path, "due to:", ex
617   
618def remove_domain(domain, force=False):
619    """Remove a domain. If force is True, domain is removed even
620    if it is not empty (contains files)."""
621    directory = localpath(domain)
622    if force:
623        import shutil
624        shutil.rmtree(directory)
625    else:
626        os.rmdir(directory)
627
628def listdomains():
629    """List all file domains in the local repository."""
630    dir = localpath()
631    _create_path(dir)
632    files = [ a for a in os.listdir(dir) ]
633    ok = []
634    for file in files:
635        if os.path.isdir(os.path.join(dir, file)):
636            ok.append(file)
637    return ok
638
639def info(domain, filename):
640    """Returns info of a file in a local repository."""
641    target = localpath(domain, filename)
642    return _open_file_info(target + '.info')
643
644def allinfo(domain):
645    """Goes through all files in a domain on a local repository and returns a
646    dictionary, where keys are names of the files and values are their
647    information."""
648    files = listfiles(domain)
649    dic = {}
650    for filename in files:
651        target = localpath(domain, filename)
652        dic[filename] = info(domain, target)
653    return dic
654
655def needs_update(domain, filename, serverfiles=None):
656    """True if a file does not exist in the local repository
657    or if there is a newer version on the server."""
658    if serverfiles == None: serverfiles = ServerFiles()
659    if filename not in listfiles(domain):
660        return True
661    dt_fmt = "%Y-%m-%d %H:%M:%S"
662    dt_local = datetime.datetime.strptime(
663        info(domain, filename)["datetime"][:19], dt_fmt)
664    dt_server = datetime.datetime.strptime(
665        serverfiles.info(domain, filename)["datetime"][:19], dt_fmt)
666    return dt_server > dt_local
667
668def update(domain, filename, serverfiles=None, **kwargs):
669    """Downloads the corresponding file from the server and places it in
670    the local repository, but only if the server copy of the file is newer
671    or the local copy does not exist. An optional  :class:`ServerFiles` object
672    can be passed for authenticated access.
673    """
674    if serverfiles == None: serverfiles = ServerFiles()
675    if needs_update(domain, filename, serverfiles=serverfiles):
676        download(domain, filename, serverfiles=serverfiles, **kwargs)
677       
678def _searchinfo():
679    domains = listdomains()
680    infos = {}
681    for dom in domains:
682        dominfo = allinfo(dom)
683        for a,b in dominfo.items():
684            infos[(dom, a)] = b
685    return infos
686
687def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
688    """
689    sstrings contain a list of search strings
690    """
691    found = []
692
693    for (dom,fn),info in si.items():
694        target = ""
695        if inTag: target += " ".join(info['tags'])
696        if inTitle: target += info['title']
697        if inName: target += fn
698        if not caseSensitive: target = target.lower()
699
700        match = True
701        for s in sstrings:
702            if not caseSensitive:
703                s = s.lower()
704            if s not in target:
705                match= False
706                break
707               
708        if match:
709            found.append((dom,fn))   
710       
711    return found
712
713
714def search(sstrings, **kwargs):
715    """Search for files in the local repository where all substrings in a list
716    are contained in at least one chosen field (tag, title, name). Return a
717    list of tuples: first tuple element is the domain of the file, second
718    its name."""
719    si = _searchinfo()
720    return _search(si, sstrings, **kwargs)
721
722
723def sizeformat(size):
724    """
725    >>> sizeformat(256)
726    256 bytes
727    >>> sizeformat(1024)
728    1.0 KB
729    >>> sizeformat(1.5 * 2 ** 20)
730    1.5 MB
731
732    """
733    for unit in ['bytes', 'KB', 'MB', 'GB', 'TB']:
734        if size < 1024.0:
735            if unit == "bytes":
736                return "%1.0f %s" % (size, unit)
737            else:
738                return "%3.1f %s" % (size, unit)
739        size /= 1024.0
740    return "%.1f PB" % size
741
742
743class DownloadProgress(ConsoleProgressBar):
744    redirect = None
745    lock = threading.RLock()
746
747    def __init__(self, filename, size):
748        print "Downloading", filename
749        ConsoleProgressBar.__init__(self, "progress:", 20)
750        self.size = size
751        self.starttime = time.time()
752        self.speed = 0.0
753
754    def sizeof_fmt(self, num):
755        return sizeformat(num)
756
757    def getstring(self):
758        elapsed = max(time.time() - self.starttime, 0.1)
759        speed = max(int(self.state * self.size / 100.0 / elapsed), 1)
760        eta = (100 - self.state) * self.size / 100.0 / speed
761        return ConsoleProgressBar.getstring(self) + \
762               %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size),
763                                               self.sizeof_fmt(speed),
764                                               eta / 60, eta % 60)
765
766    def __call__(self, *args, **kwargs):
767        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
768        if self.redirect:
769            self.redirect(self.state)
770        return ret
771
772    class RedirectContext(object):
773        def __enter__(self):
774            DownloadProgress.lock.acquire()
775            return DownloadProgress
776
777        def __exit__(self, ex_type, value, tb):
778            DownloadProgress.redirect = None
779            DownloadProgress.lock.release()
780            return False
781
782    @classmethod
783    def setredirect(cls, redirect):
784        cls.redirect = staticmethod(redirect)
785        return cls.RedirectContext()
786
787    @classmethod
788    def __enter__(cls):
789        cls.lock.acquire()
790        return cls
791
792    @classmethod
793    def __exit__(cls, exc_type, exc_value, traceback):
794        cls.lock.release()
795        return False
796
797
798def consoleupdate(domains=None, searchstr="essential"):
799    domains = domains or listdomains()
800    sf = ServerFiles()
801    info = dict((d, sf.allinfo(d)) for d in domains)
802    def searchmenu():
803        def printmenu():
804            print "\tSearch tags:", search
805            print "\t1. Add tag."
806            print "\t2. Clear tags."
807            print "\t0. Return to main menu."
808            return raw_input("\tSelect option:")
809        search = searchstr
810        while True:
811            response = printmenu().strip()
812            if response == "1":
813                search += " " + raw_input("\tType new tag/tags:")
814            elif response == "2":
815                search = ""
816            elif response == "0":
817                break
818            else:
819                print "\tUnknown option!"
820        return search
821
822    def filemenu(searchstr=""):
823        files = [None]
824        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
825            print "\t%i." % (i + 1), info[dom][file]["title"]
826            files.append((dom, file))
827        print "\t0. Return to main menu."
828        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
829        while True:
830            response = raw_input("\tAction:").strip()
831            if response == "0":
832                break
833            try:
834                action, num = response.split(None, 1)
835                num = int(num)
836            except Exception, ex:
837                print "Unknown option!"
838                continue
839            try:
840                if action.lower() == "d":
841                    download(*(files[num]))
842                    print "\tSuccsessfully downloaded", files[num][-1]
843            except Exception, ex:
844                print "Error occured!", ex
845
846    def printmenu():
847        print "Update database main menu:"
848        print "1. Enter search tags (refine search)."
849        print "2. Print matching available files."
850        print "3. Print all available files."
851        print "4. Update all local files."
852        print "0. Exit."
853        return raw_input("Select option:")
854   
855    while True:
856        try:
857            response = printmenu().strip()
858            if response == "1":
859                searchstr = searchmenu()
860            elif response == "2":
861                filemenu(searchstr)
862            elif response == "3":
863                filemenu("")
864            elif response == "4":
865                update_local_files()
866            elif response == "0":
867                break
868            else:
869                print "Unknown option!"
870        except Exception, ex:
871            print "Error occured:", ex
872
873def update_local_files(verbose=True):
874    sf = ServerFiles()
875    for domain, filename in search(""):
876        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
877        if not uptodate:
878            download(domain, filename, sf)
879        if verbose:
880            print filename, "Ok" if uptodate else "Updated"
881
882def update_by_tags(tags=["essential"], domains=[], verbose=True):
883    sf = ServerFiles()
884    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
885        if domains and domain not in domain:
886            continue
887        if os.path.exists(localpath(domain, filename)+".info"):
888            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
889        else:
890            uptodate = False
891        if not uptodate:
892            download(domain, filename, sf)
893        if verbose:
894            print filename, "Ok" if uptodate else "Updated"
895           
896def _example(myusername, mypassword):
897
898    locallist = listfiles('test')
899    for l in locallist:
900        print info('test', l)
901
902    s = ServerFiles()
903
904    print "testing connection - public"
905    print "AN", s.index()
906
907    #login as an authenticated user
908    s = ServerFiles(username=myusername, password=mypassword)
909   
910    """
911    print "Server search 1"
912    import time
913    t = time.time()
914    print s.search(["rat"])
915    print time.time() - t
916
917    t = time.time()
918    print s.search(["human", "ke"])
919    print time.time() - t
920    """
921
922    print "testing connection - private"
923    print "AN", s.index()
924
925    #create domain
926    try: 
927        s.create_domain("test") 
928    except:
929        print "Failed to create the domain"
930        pass
931
932    files = s.listfiles('test')
933    print "Files in test", files
934
935    print "uploading"
936
937    #upload this file - save it by a different name
938    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
939    #make it public
940    s.unprotect('test', 'osf-test.py')
941
942    #login anonymously
943    s = ServerFiles()
944
945    #list files in the domain "test"
946    files = s.listfiles('test')
947    print "ALL FILES:", files
948
949    for f in files:
950        fi = s.info('test', f) 
951        print "--------------------------------------", f
952        print "INFO", fi
953        print s.downloadFH('test', f).read()[:100] #show first 100 characters
954        print "--------------------------------------"
955
956    #login as an authenticated user
957    s = ServerFiles(username=myusername, password=mypassword)
958
959    print s.listdomains()
960
961    s.remove('test', 'osf-test.py')
962
963    s = ServerFiles()
964
965    print s.listdomains()
966
967
968if __name__ == '__main__':
969    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.