source: orange/orange/Orange/misc/serverfiles.py @ 7744:ccb8bd3760c0

Revision 7744:ccb8bd3760c0, 28.2 KB checked in by markotoplak, 3 years ago (diff)

serverfiles: update and needs_update take an optional ServerFiles object instead of access_code. orngServerFiles modified for backwards compatibility.

Line 
1"""
2Server files allows users to download files from a common
3repository residing on the Orange server. It was designed to simplify
4the download and updates of external data sources for Orange Genomics add-on.
5Furthermore, an authenticated user can also manage the repository files with
6this module.
7
8Orange server file repository was created to store large files that do not
9come with Orange installation, but may be required from the user when
10running specific Orange functions. A typical example is Orange Bioinformatics
11package, which relies on large data files storing genome information.
12These do not come pre-installed, but are rather downloaded from the server
13when needed and stored in the local repository. The module provides low-level
14functionality to manage these files, and is used by Orange modules to locate
15the files from the local repository and update/download them when and if needed.
16
17Each managed file is described by domain and the file name.
18Domains are like directories - a place where files are put in.
19
20Domain should consist of less than 255 alphanumeric ASCII characters, whereas
21filenames can be arbitary long and can contain any ASCII character (including
22"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
23domain and filenames. Files can be protected or not. Protected files can
24only be accessed by authenticated users
25
26Local file management
27=====================
28
29The files are saved under Orange's settings directory,
30subdirectory buffer/bigfiles. Each domain is a subdirectory.
31A corresponding info
32file bearing the same name and an extension ".info" is created
33with every download. Info files
34contain title, tags, size and date and time of the file.
35
36.. autofunction:: allinfo
37
38.. autofunction:: download
39
40.. autofunction:: info
41
42.. autofunction:: listdomains
43
44.. autofunction:: listfiles
45
46.. autofunction:: localpath
47
48.. autofunction:: localpath_download
49
50.. autofunction:: needs_update
51
52.. autofunction:: remove
53
54.. autofunction:: remove_domain
55
56.. autofunction:: search
57
58.. autofunction:: update
59
60
61Remote file management
62======================
63
64.. autoclass:: ServerFiles
65    :members:
66
67Examples
68========
69
70.. _serverfiles1.py: code/serverfiles1.py
71.. _serverfiles2.py: code/serverfiles2.py
72
73Listing local files, files from the repository and downloading all available files from domain "demo" (`serverfiles1.py`_).
74
75.. literalinclude:: code/serverfiles1.py
76
77A possible output (it depends on the current repository state)::
78
79    My files []
80    Repository files ['orngServerFiles.py', 'urllib2_file.py']
81    Downloading all files in domain 'test'
82    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
83    Downloading orngServerFiles.py
84    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
85    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
86    Downloading urllib2_file.py
87    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
88    My files after download ['urllib2_file.py', 'orngServerFiles.py']
89    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
90
91A domain with a simple file can be built as follows (`serverfiles2.py`_). Of course,
92the username and password should be valid.
93
94.. literalinclude:: code/serverfiles2.py
95
96A possible output::
97
98    Uploaded.
99    Non-authenticated users see: ['']
100    Authenticated users see: ['titanic.tab']
101    Non-authenticated users now see: ['titanic.tab']
102    orngServerFiles.py file info:
103    {'datetime': '2011-03-15 13:18:53.029000',
104     'size': '45112',
105     'tags': ['basic', 'data set'],
106     'title': 'A sample .tab file'}
107
108"""
109
110import sys
111import socket
112
113# timeout in seconds
114timeout = 120
115socket.setdefaulttimeout(timeout)
116
117import urllib2
118import base64
119
120import urllib2_file 
121#switch to poster in the future
122#import poster.streaminghttp as psh
123#import poster.encode
124
125from orngMisc import ConsoleProgressBar
126import time, threading
127
128import os
129import shutil
130import glob
131import datetime
132import tempfile
133
134#defserver = "localhost:9999/"
135defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
136
137def _parseFileInfo(fir, separ="|||||"):
138    """
139    Parses file info from server.
140    """
141    l= fir.split(separ)
142    fi = {}
143    fi["size"] = l[0]
144    fi["datetime"] = l[1]
145    fi["title"] = l[2]
146    fi["tags"] = l[3].split(";")
147    return fi
148
149def _open_file_info(fname): #no outer usage
150    f = open(fname, 'rt')
151    info = _parseFileInfo(f.read(), separ='\n')
152    f.close()
153    return info
154
155def _save_file_info(fname, info): #no outer usage
156    f = open(fname, 'wt')
157    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
158    f.close()
159
160def _parseList(fl):
161    return fl.split("|||||")
162
163def _parseAllFileInfo(afi):
164    separf = "[[[[["
165    separn = "====="
166    fis = afi.split(separf)
167    out = []
168    for entry in fis:
169        if entry != "":
170            name, info = entry.split(separn)
171            out.append((name, _parseFileInfo(info)))
172
173    return dict(out)
174
175def _create_path_for_file(target):
176    try:
177        os.makedirs(os.path.dirname(target))
178    except OSError:
179        pass
180
181def _create_path(target):
182    try:
183        os.makedirs(target)
184    except OSError:
185        pass
186
187def localpath(domain=None, filename=None):
188    """Return a path for the domain in the local repository. If
189    filename is given, return a path to corresponding file."""
190    import orngEnviron
191    if not domain:
192        return os.path.join(orngEnviron.directoryNames["bufferDir"],
193            "bigfiles")
194    if filename:
195        return os.path.join(orngEnviron.directoryNames["bufferDir"],
196            "bigfiles", domain, filename)
197    else:
198        return os.path.join(orngEnviron.directoryNames["bufferDir"],
199            "bigfiles", domain)
200
201class ServerFiles(object):
202    """
203    To work with the repository, you need to create an instance of
204    ServerFiles object. To access the repository as an authenticated user, a
205    username and password should be passed to the constructor. All password
206    protected operations and transfers are secured by SSL; this secures
207    both password and content.
208
209    Repository files are set as protected when first uploaded: only
210    authenticated users can see them. They need to be unprotected for
211    public use.
212    """
213
214    def __init__(self, username=None, password=None, server=None, access_code=None):
215        """
216        Creates a ServerFiles instance. Pass your username and password
217        to use the repository as an authenticated user. If you want to use
218        your access code (as an non-authenticated user), pass it also.
219        """
220        if not server:
221            server = defserver
222        self.server = server
223        self.secureroot = 'https://' + self.server + 'private/'
224        self.publicroot = 'http://' + self.server + 'public/'
225        self.username = username
226        self.password = password
227        self.access_code = access_code
228        self.searchinfo = None
229
230    def _getOpener(self):
231        #commented lines are for poster 0.6
232        #handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
233        #opener = urllib2.build_opener(*handlers)
234        opener = urllib2.build_opener()
235        return opener
236 
237    def upload(self, domain, filename, file, title="", tags=[]):
238        """ Uploads a file "file" to the domain where it is saved with filename
239        "filename". If file does not exist yet, set it as protected. Parameter
240        file can be a file handle open for reading or a file name.
241        """
242        if isinstance(file, basestring):
243            file = open(file, 'rb')
244
245        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
246        return self._open('upload', data)
247
248    def create_domain(self, domain):
249        """Create a server domain."""
250        return self._open('createdomain', { 'domain': domain })
251
252    def remove_domain(self, domain, force=False):
253        """Remove a domain. If force is True, domain is removed even
254        if it is not empty (contains files)."""
255        data = { 'domain': domain }
256        if force:
257            data['force'] = True
258        return self._open('removedomain', data)
259
260    def remove(self, domain, filename):
261        """Remove a file from the server repository."""
262        return self._open('remove', { 'domain': domain, 'filename': filename })
263
264    def unprotect(self, domain, filename):
265        """Put a file into public use."""
266        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
267
268    def protect(self, domain, filename, access_code="1"):
269        """Hide file from non-authenticated users. If an access code (string)
270        is passed, the file will be available to authenticated users and
271        non-authenticated users with that access code."""
272        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
273
274    def protection(self, domain, filename):
275        """Return file protection. Legend: "0" - public use,
276        "1" - for authenticated users only, anything else
277        represents a specific access code.
278        """
279        return self._open('protection', { 'domain': domain, 'filename': filename })
280   
281    def listfiles(self, domain):
282        """List all files in a repository domain."""
283        return _parseList(self._open('list', { 'domain': domain }))
284
285    def listdomains(self):
286        """List all domains on repository."""
287        return _parseList(self._open('listdomains', {}))
288
289    def downloadFH(self, *args, **kwargs):
290        """Return open file handle of requested file from the server repository given the domain and the filename."""
291        if self._authen(): return self.secdownloadFH(*args, **kwargs)
292        else: return self.pubdownloadFH(*args, **kwargs)
293
294    def download(self, domain, filename, target, callback=None):
295        """
296        Downloads file from the repository to a given target name. Callback
297        can be a function without arguments. It will be called once for each
298        downloaded percent of file: 100 times for the whole file.
299        """
300        _create_path_for_file(target)
301
302        fdown = self.downloadFH(domain, filename)
303        size = int(fdown.headers.getheader('content-length'))
304
305        f = tempfile.TemporaryFile()
306 
307        chunksize = 1024*8
308        lastchunkreport= 0.0001
309
310        readb = 0
311        while 1:
312            buf = fdown.read(chunksize)
313            readb += len(buf)
314
315            while float(readb)/size > lastchunkreport+0.01:
316                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
317                lastchunkreport += 0.01
318                if callback:
319                    callback()
320            if not buf:
321                break
322            f.write(buf)
323
324        fdown.close()
325        f.seek(0)
326
327        shutil.copyfileobj(f, open(target, "wb"))
328
329        if callback:
330            callback()
331
332    def _searchinfo(self):
333        domains = self.listdomains()
334        infos = {}
335        for dom in domains:
336            dominfo = self.allinfo(dom)
337            for a,b in dominfo.items():
338                infos[(dom, a)] = b
339        return infos
340
341    def search(self, sstrings, **kwargs):
342        """
343        Search for files on the repository where all substrings in a list
344        are contained in at least one choosen field (tag, title, name). Return
345        a list of tuples: first tuple element is the file's domain, second its
346        name. As for now the search is performed locally, therefore
347        information on files in repository is transfered on first call of
348        this function.
349        """
350        if not self.searchinfo:
351            self.searchinfo = self._searchinfo()
352        return _search(self.searchinfo, sstrings, **kwargs)
353
354    def info(self, domain, filename):
355        """Return a dictionary containing repository file info.
356        Keys: title, tags, size, datetime."""
357        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
358
359    def downloadFH(self, domain, filename):
360        """Return a file handle to the file that we would like to download."""
361        return self._handle('download', { 'domain': domain, 'filename': filename })
362
363    def list(self, domain):
364        return _parseList(self._open('list', { 'domain': domain }))
365
366    def listdomains(self):
367        """List all domains on repository."""
368        return _parseList(self._open('listdomains', {}))
369
370    def allinfo(self, domain):
371        """Go through all accessible files in a given domain and return a
372        dictionary, where key is file's name and value its info.
373        """
374        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
375
376    def index(self):
377        return self._open('index', {})
378
379    def _authen(self):
380        """
381        Did the user choose authentication?
382        """
383        if self.username and self.password:
384            return True
385        else:
386            return False
387
388    def _server_request(self, root, command, data, repeat=2):
389        def do():
390            opener = self._getOpener()
391            #the next lines work for poster 0.6.0
392            #datagen, headers = poster.encode.multipart_encode(data)
393            #request = urllib2.Request(root+command, datagen, headers)
394
395            if data:
396                request = urllib2.Request(root+command, data)
397            else:
398                request = urllib2.Request(root+command)
399
400            #directy add authorization headers
401            if self._authen():
402                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
403                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
404           
405            return opener.open(request)
406        if repeat <= 0:
407            return do()
408        else:
409            try:
410                return do()
411            except:
412                return self._server_request(root, command, data, repeat=repeat-1)
413   
414    def _handle(self, command, data):
415        data2 = self._addAccessCode(data)
416        addr = self.publicroot
417        if self._authen():
418            addr = self.secureroot
419        return self._server_request(addr, command, data)
420
421    def _open(self, command, data):
422        return self._handle(command, data).read()
423
424    def _addAccessCode(self, data):
425        if self.access_code != None:
426            data = data.copy()
427            data["access_code"] = self.access_code
428        return data
429
430def download(domain, filename, serverfiles=None, callback=None, 
431    extract=True, verbose=True):
432    """Downloads file from the repository to local orange installation.
433    To download files as an authenticated user you should also pass an
434    instance of ServerFiles class. Callback can be a function without
435    arguments. It will be called once for each downloaded percent of
436    file: 100 times for the whole file."""
437
438    if not serverfiles:
439        serverfiles = ServerFiles()
440
441    info = serverfiles.info(domain, filename)
442    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
443    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
444    target = localpath(domain, filename)
445    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
446    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
447   
448    #file saved, now save info file
449
450    _save_file_info(target + '.info', info)
451   
452    if extract:
453        import tarfile, gzip, shutil
454        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
455            f = tarfile.open(target + ".tmp")
456            f.extractall(localpath(domain))
457            shutil.copyfile(target + ".tmp", target)
458        if filename.endswith(".tar.gz"):
459            f = tarfile.open(target + ".tmp")
460            try:
461                os.mkdir(target)
462            except Exception:
463                pass
464            f.extractall(target)
465        elif specialtags.get("#compression") == "gz":
466            f = gzip.open(target + ".tmp")
467            shutil.copyfileobj(f, open(target, "wb"))
468        f.close()
469        os.remove(target + ".tmp")
470
471    if type(callback) == DownloadProgress:
472        callback.finish()
473
474def localpath_download(domain, filename, **kwargs):
475    """
476    Return local path for the given domain and file. If file does not exist,
477    download it. Additional arguments are passed to the :obj:`download` function.
478    """
479    pathname = localpath(domain, filename)
480    if not os.path.exists(pathname):
481        download(domain, filename, **kwargs)
482    return pathname
483
484def listfiles(domain):
485    """List all files from a domain in a local repository."""
486    dir = localpath(domain)
487    try:
488        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
489    except:
490        files = []
491    okfiles = []
492
493    for file in files:
494        #if file to exists without info
495        if os.path.exists(os.path.join(dir,file[:-5])):
496            #check info format - needs to be valid
497            try:
498                _open_file_info(os.path.join(dir,file))
499                okfiles.append(file[:-5])
500            except:
501                pass
502
503    return okfiles
504
505def remove(domain, filename):
506    """Remove a file from local repository."""
507    filename = localpath(domain, filename)
508    import shutil
509   
510    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
511    todelete = [filename, filename + ".info"] 
512    if "#files" in specialtags:
513        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
514#    print todelete
515    for path in todelete:
516        try:
517            if os.path.isdir(path):
518                shutil.rmtree(path)
519            elif os.path.isfile(path):
520                os.remove(path)
521        except OSError, ex:
522            print "Failed to delete", path, "due to:", ex
523   
524def remove_domain(domain, force=False):
525    """Remove a domain. If force is True, domain is removed even
526    if it is not empty (contains files)."""
527    directory = localpath(domain)
528    if force:
529        import shutil
530        shutil.rmtree(directory)
531    else:
532        os.rmdir(directory)
533
534def listdomains():
535    """List all file domains in the local repository."""
536    dir = localpath()
537    _create_path(dir)
538    files = [ a for a in os.listdir(dir) ]
539    ok = []
540    for file in files:
541        if os.path.isdir(os.path.join(dir, file)):
542            ok.append(file)
543    return ok
544
545def info(domain, filename):
546    """Returns info of a file in a local repository."""
547    target = localpath(domain, filename)
548    return _open_file_info(target + '.info')
549
550def allinfo(domain):
551    """Goes through all files in a domain on a local repository and returns a
552    dictionary, where keys are names of the files and values are their
553    information."""
554    files = listfiles(domain)
555    dic = {}
556    for filename in files:
557        target = localpath(domain, filename)
558        dic[filename] = info(domain, target)
559    return dic
560
561def needs_update(domain, filename, serverfiles=None):
562    """True if a file does not exist in the local repository
563    or if there is a newer version on the server."""
564    if serverfiles == None: serverfiles = ServerFiles()
565    if filename not in listfiles(domain):
566        return True
567    dt_fmt = "%Y-%m-%d %H:%M:%S"
568    dt_local = datetime.datetime.strptime(
569        info(domain, filename)["datetime"][:19], dt_fmt)
570    dt_server = datetime.datetime.strptime(
571        serverfiles.info(domain, filename)["datetime"][:19], dt_fmt)
572    return dt_server > dt_local
573
574def update(domain, filename, serverfiles=None, **kwargs):
575    """Downloads the corresponding file from the server and places it in
576    the local repository, but only if the server copy of the file is newer
577    or the local copy does not exist. An optional  :class:`ServerFiles` object
578    can be passed for authenticated access.
579    """
580    if serverfiles == None: serverfiles = ServerFiles()
581    if needs_update(domain, filename, serverfiles=serverfiles):
582        download(domain, filename, serverfiles=serverfiles, **kwargs)
583       
584def _searchinfo():
585    domains = listdomains()
586    infos = {}
587    for dom in domains:
588        dominfo = allinfo(dom)
589        for a,b in dominfo.items():
590            infos[(dom, a)] = b
591    return infos
592
593def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
594    """
595    sstrings contain a list of search strings
596    """
597    found = []
598
599    for (dom,fn),info in si.items():
600        target = ""
601        if inTag: target += " ".join(info['tags'])
602        if inTitle: target += info['title']
603        if inName: target += fn
604        if not caseSensitive: target = target.lower()
605
606        match = True
607        for s in sstrings:
608            if not caseSensitive:
609                s = s.lower()
610            if s not in target:
611                match= False
612                break
613               
614        if match:
615            found.append((dom,fn))   
616       
617    return found
618
619def search(sstrings, **kwargs):
620    """Search for files in the local repository where all substrings in a list
621    are contained in at least one chosen field (tag, title, name). Return a
622    list of tuples: first tuple element is the domain of the file, second
623    its name."""
624    si = _searchinfo()
625    return _search(si, sstrings, **kwargs)
626
627class DownloadProgress(ConsoleProgressBar):
628    redirect = None
629    lock = threading.RLock()
630    def sizeof_fmt(num):
631        for x in ['bytes','KB','MB','GB','TB']:
632            if num < 1024.0:
633                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
634            num /= 1024.0
635           
636    def __init__(self, filename, size):
637        print "Downloading", filename
638        ConsoleProgressBar.__init__(self, "progress:", 20)
639        self.size = size
640        self.starttime = time.time()
641        self.speed = 0.0
642
643    def sizeof_fmt(self, num):
644        for x in ['bytes','KB','MB','GB','TB']:
645            if num < 1024.0:
646                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
647            num /= 1024.0
648
649    def getstring(self):
650        speed = int(self.state * self.size / 100.0 / (time.time() - self.starttime))
651        eta = (100 - self.state) * self.size / 100.0 / speed
652        return ConsoleProgressBar.getstring(self) + %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size), self.sizeof_fmt(speed), eta/60, eta%60)
653       
654    def __call__(self, *args, **kwargs):
655        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
656        if self.redirect:
657            self.redirect(self.state)
658        return ret
659   
660    class RedirectContext(object):
661        def __enter__(self):
662            DownloadProgress.lock.acquire()
663            return DownloadProgress
664       
665        def __exit__(self, ex_type, value, tb):
666            DownloadProgress.redirect = None
667            DownloadProgress.lock.release()
668            return False
669       
670    @classmethod
671    def setredirect(cls, redirect):
672        cls.redirect = staticmethod(redirect)
673        return cls.RedirectContext()
674   
675    @classmethod
676    def __enter__(cls):
677        cls.lock.acquire()
678        return cls
679   
680    @classmethod
681    def __exit__(cls, exc_type, exc_value, traceback):
682        cls.lock.release()
683        return False
684
685def consoleupdate(domains=None, searchstr="essential"):
686    domains = domains or listdomains()
687    sf = ServerFiles()
688    info = dict((d, sf.allinfo(d)) for d in domains)
689    def searchmenu():
690        def printmenu():
691            print "\tSearch tags:", search
692            print "\t1. Add tag."
693            print "\t2. Clear tags."
694            print "\t0. Return to main menu."
695            return raw_input("\tSelect option:")
696        search = searchstr
697        while True:
698            response = printmenu().strip()
699            if response == "1":
700                search += " " + raw_input("\tType new tag/tags:")
701            elif response == "2":
702                search = ""
703            elif response == "0":
704                break
705            else:
706                print "\tUnknown option!"
707        return search
708
709    def filemenu(searchstr=""):
710        files = [None]
711        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
712            print "\t%i." % (i + 1), info[dom][file]["title"]
713            files.append((dom, file))
714        print "\t0. Return to main menu."
715        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
716        while True:
717            response = raw_input("\tAction:").strip()
718            if response == "0":
719                break
720            try:
721                action, num = response.split(None, 1)
722                num = int(num)
723            except Exception, ex:
724                print "Unknown option!"
725                continue
726            try:
727                if action.lower() == "d":
728                    download(*(files[num]))
729                    print "\tSuccsessfully downloaded", files[num][-1]
730            except Exception, ex:
731                print "Error occured!", ex
732
733    def printmenu():
734        print "Update database main menu:"
735        print "1. Enter search tags (refine search)."
736        print "2. Print matching available files."
737        print "3. Print all available files."
738        print "4. Update all local files."
739        print "0. Exit."
740        return raw_input("Select option:")
741   
742    while True:
743        try:
744            response = printmenu().strip()
745            if response == "1":
746                searchstr = searchmenu()
747            elif response == "2":
748                filemenu(searchstr)
749            elif response == "3":
750                filemenu("")
751            elif response == "4":
752                update_local_files()
753            elif response == "0":
754                break
755            else:
756                print "Unknown option!"
757        except Exception, ex:
758            print "Error occured:", ex
759
760def update_local_files(verbose=True):
761    sf = ServerFiles()
762    for domain, filename in search(""):
763        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
764        if not uptodate:
765            download(domain, filename, sf)
766        if verbose:
767            print filename, "Ok" if uptodate else "Updated"
768
769def update_by_tags(tags=["essential"], domains=[], verbose=True):
770    sf = ServerFiles()
771    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
772        if domains and domain not in domain:
773            continue
774        if os.path.exists(localpath(domain, filename)+".info"):
775            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
776        else:
777            uptodate = False
778        if not uptodate:
779            download(domain, filename, sf)
780        if verbose:
781            print filename, "Ok" if uptodate else "Updated"
782           
783def _example(myusername, mypassword):
784
785    locallist = listfiles('test')
786    for l in locallist:
787        print info('test', l)
788
789    s = ServerFiles()
790
791    print "testing connection - public"
792    print "AN", s.index()
793
794    #login as an authenticated user
795    s = ServerFiles(username=myusername, password=mypassword)
796   
797    """
798    print "Server search 1"
799    import time
800    t = time.time()
801    print s.search(["rat"])
802    print time.time() - t
803
804    t = time.time()
805    print s.search(["human", "ke"])
806    print time.time() - t
807    """
808
809    print "testing connection - private"
810    print "AN", s.index()
811
812    #create domain
813    try: 
814        s.create_domain("test") 
815    except:
816        print "Failed to create the domain"
817        pass
818
819    files = s.listfiles('test')
820    print "Files in test", files
821
822    print "uploading"
823
824    #upload this file - save it by a different name
825    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
826    #make it public
827    s.unprotect('test', 'osf-test.py')
828
829    #login anonymously
830    s = ServerFiles()
831
832    #list files in the domain "test"
833    files = s.listfiles('test')
834    print "ALL FILES:", files
835
836    for f in files:
837        fi = s.info('test', f) 
838        print "--------------------------------------", f
839        print "INFO", fi
840        print s.downloadFH('test', f).read()[:100] #show first 100 characters
841        print "--------------------------------------"
842
843    #login as an authenticated user
844    s = ServerFiles(username=myusername, password=mypassword)
845
846    print s.listdomains()
847
848    s.remove('test', 'osf-test.py')
849
850    s = ServerFiles()
851
852    print s.listdomains()
853
854
855if __name__ == '__main__':
856    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.