source: orange/orange/Orange/misc/serverfiles.py @ 7742:f8f8bdab658d

Revision 7742:f8f8bdab658d, 28.3 KB checked in by markotoplak, 3 years ago (diff)

Moved orngServerFiles to a new hierarchy.

Line 
1"""
2Server files allows users to download files from a common
3repository residing on the Orange server. It was designed to simplify
4the download and updates of external data sources for Orange Genomics add-on.
5Furthermore, an authenticated user can also manage the repository files with
6this module.
7
8Orange server file repository was created to store large files that do not
9come with Orange installation, but may be required from the user when
10running specific Orange functions. A typical example is Orange Bioinformatics
11package, which relies on large data files storing genome information.
12These do not come pre-installed, but are rather downloaded from the server
13when needed and stored in the local repository. The module provides low-level
14functionality to manage these files, and is used by Orange modules to locate
15the files from the local repository and update/download them when and if needed.
16
17Each managed file is described by domain and the file name.
18Domains are like directories - a place where files are put in.
19
20Domain should consist of less than 255 alphanumeric ASCII characters, whereas
21filenames can be arbitary long and can contain any ASCII character (including
22"" ~ . \ / { }). Please, refrain from using not-ASCII character both in
23domain and filenames. Files can be protected or not. Protected files can
24only be accessed by authenticated users
25
26Local file management
27=====================
28
29The files are saved under Orange's settings directory,
30subdirectory buffer/bigfiles. Each domain is a subdirectory.
31A corresponding info
32file bearing the same name and an extension ".info" is created
33with every download. Info files
34contain title, tags, size and date and time of the file.
35
36.. autofunction:: allinfo
37
38.. autofunction:: download
39
40.. autofunction:: info
41
42.. autofunction:: listdomains
43
44.. autofunction:: listfiles
45
46.. autofunction:: localpath
47
48.. autofunction:: localpath_download
49
50.. autofunction:: needs_update
51
52.. autofunction:: remove
53
54.. autofunction:: remove_domain
55
56.. autofunction:: search
57
58.. autofunction:: update
59
60
61Remote file management
62======================
63
64.. autoclass:: ServerFiles
65    :members:
66
67Examples
68========
69
70.. _serverfiles1.py: code/serverfiles1.py
71.. _serverfiles2.py: code/serverfiles2.py
72
73Listing local files, files from the repository and downloading all available files from domain "demo" (`serverfiles1.py`_).
74
75.. literalinclude:: code/serverfiles1.py
76
77A possible output (it depends on the current repository state)::
78
79    My files []
80    Repository files ['orngServerFiles.py', 'urllib2_file.py']
81    Downloading all files in domain 'test'
82    Datetime for orngServerFiles.py 2008-08-20 12:25:54.624000
83    Downloading orngServerFiles.py
84    progress: ===============>100%  10.7 KB       47.0 KB/s    0:00 ETA
85    Datetime for urllib2_file.py 2008-08-20 12:25:54.827000
86    Downloading urllib2_file.py
87    progress: ===============>100%  8.5 KB       37.4 KB/s    0:00 ETA
88    My files after download ['urllib2_file.py', 'orngServerFiles.py']
89    My domains ['KEGG', 'gene_sets', 'dictybase', 'NCBI_geneinfo', 'GO', 'miRNA', 'demo', 'Taxonomy', 'GEO']
90
91A domain with a simple file can be built as follows (`serverfiles2.py`_). Of course,
92the username and password should be valid.
93
94.. literalinclude:: code/serverfiles2.py
95
96A possible output::
97
98    Uploaded.
99    Non-authenticated users see: ['']
100    Authenticated users see: ['titanic.tab']
101    Non-authenticated users now see: ['titanic.tab']
102    orngServerFiles.py file info:
103    {'datetime': '2011-03-15 13:18:53.029000',
104     'size': '45112',
105     'tags': ['basic', 'data set'],
106     'title': 'A sample .tab file'}
107
108"""
109
110import sys
111import socket
112
113# timeout in seconds
114timeout = 120
115socket.setdefaulttimeout(timeout)
116
117import urllib2
118import base64
119
120import urllib2_file 
121#switch to poster in the future
122#import poster.streaminghttp as psh
123#import poster.encode
124
125from orngMisc import ConsoleProgressBar
126import time, threading
127
128import os
129import shutil
130import glob
131import datetime
132import tempfile
133
134#defserver = "localhost:9999/"
135defserver = "asterix.fri.uni-lj.si/orngServerFiles/"
136
137def _parseFileInfo(fir, separ="|||||"):
138    """
139    Parses file info from server.
140    """
141    l= fir.split(separ)
142    fi = {}
143    fi["size"] = l[0]
144    fi["datetime"] = l[1]
145    fi["title"] = l[2]
146    fi["tags"] = l[3].split(";")
147    return fi
148
149def _open_file_info(fname): #no outer usage
150    f = open(fname, 'rt')
151    info = _parseFileInfo(f.read(), separ='\n')
152    f.close()
153    return info
154
155def _save_file_info(fname, info): #no outer usage
156    f = open(fname, 'wt')
157    f.write('\n'.join([info['size'], info['datetime'], info['title'], ';'.join(info['tags'])]))
158    f.close()
159
160def _parseList(fl):
161    return fl.split("|||||")
162
163def _parseAllFileInfo(afi):
164    separf = "[[[[["
165    separn = "====="
166    fis = afi.split(separf)
167    out = []
168    for entry in fis:
169        if entry != "":
170            name, info = entry.split(separn)
171            out.append((name, _parseFileInfo(info)))
172
173    return dict(out)
174
175def _create_path_for_file(target):
176    try:
177        os.makedirs(os.path.dirname(target))
178    except OSError:
179        pass
180
181def _create_path(target):
182    try:
183        os.makedirs(target)
184    except OSError:
185        pass
186
187def localpath(domain=None, filename=None):
188    """Return a path for the domain in the local repository. If
189    filename is given, return a path to corresponding file."""
190    import orngEnviron
191    if not domain:
192        return os.path.join(orngEnviron.directoryNames["bufferDir"],
193            "bigfiles")
194    if filename:
195        return os.path.join(orngEnviron.directoryNames["bufferDir"],
196            "bigfiles", domain, filename)
197    else:
198        return os.path.join(orngEnviron.directoryNames["bufferDir"],
199            "bigfiles", domain)
200
201class ServerFiles(object):
202    """
203    To work with the repository, you need to create an instance of
204    ServerFiles object. To access the repository as an authenticated user, a
205    username and password should be passed to the constructor. All password
206    protected operations and transfers are secured by SSL; this secures
207    both password and content.
208
209    Repository files are set as protected when first uploaded: only
210    authenticated users can see them. They need to be unprotected for
211    public use.
212    """
213
214    def __init__(self, username=None, password=None, server=None, access_code=None):
215        """
216        Creates a ServerFiles instance. Pass your username and password
217        to use the repository as an authenticated user. If you want to use
218        your access code (as an non-authenticated user), pass it also.
219        """
220        if not server:
221            server = defserver
222        self.server = server
223        self.secureroot = 'https://' + self.server + 'private/'
224        self.publicroot = 'http://' + self.server + 'public/'
225        self.username = username
226        self.password = password
227        self.access_code = access_code
228        self.searchinfo = None
229
230    def _getOpener(self):
231        #commented lines are for poster 0.6
232        #handlers = [psh.StreamingHTTPHandler, psh.StreamingHTTPRedirectHandler, psh.StreamingHTTPSHandler]
233        #opener = urllib2.build_opener(*handlers)
234        opener = urllib2.build_opener()
235        return opener
236 
237    def upload(self, domain, filename, file, title="", tags=[]):
238        """ Uploads a file "file" to the domain where it is saved with filename
239        "filename". If file does not exist yet, set it as protected. Parameter
240        file can be a file handle open for reading or a file name.
241        """
242        if isinstance(file, basestring):
243            file = open(file, 'rb')
244
245        data = {'filename': filename, 'domain': domain, 'title':title, 'tags': ";".join(tags), 'data':  file}
246        return self._open('upload', data)
247
248    def create_domain(self, domain):
249        """Create a server domain."""
250        return self._open('createdomain', { 'domain': domain })
251
252    def remove_domain(self, domain, force=False):
253        """Remove a domain. If force is True, domain is removed even
254        if it is not empty (contains files)."""
255        data = { 'domain': domain }
256        if force:
257            data['force'] = True
258        return self._open('removedomain', data)
259
260    def remove(self, domain, filename):
261        """Remove a file from the server repository."""
262        return self._open('remove', { 'domain': domain, 'filename': filename })
263
264    def unprotect(self, domain, filename):
265        """Put a file into public use."""
266        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': '0' })
267
268    def protect(self, domain, filename, access_code="1"):
269        """Hide file from non-authenticated users. If an access code (string)
270        is passed, the file will be available to authenticated users and
271        non-authenticated users with that access code."""
272        return self._open('protect', { 'domain': domain, 'filename': filename, 'access_code': access_code })
273
274    def protection(self, domain, filename):
275        """Return file protection. Legend: "0" - public use,
276        "1" - for authenticated users only, anything else
277        represents a specific access code.
278        """
279        return self._open('protection', { 'domain': domain, 'filename': filename })
280   
281    def listfiles(self, domain):
282        """List all files in a repository domain."""
283        return _parseList(self._open('list', { 'domain': domain }))
284
285    def listdomains(self):
286        """List all domains on repository."""
287        return _parseList(self._open('listdomains', {}))
288
289    def downloadFH(self, *args, **kwargs):
290        """Return open file handle of requested file from the server repository given the domain and the filename."""
291        if self._authen(): return self.secdownloadFH(*args, **kwargs)
292        else: return self.pubdownloadFH(*args, **kwargs)
293
294    def download(self, domain, filename, target, callback=None):
295        """
296        Downloads file from the repository to a given target name. Callback
297        can be a function without arguments. It will be called once for each
298        downloaded percent of file: 100 times for the whole file.
299        """
300        _create_path_for_file(target)
301
302        fdown = self.downloadFH(domain, filename)
303        size = int(fdown.headers.getheader('content-length'))
304
305        f = tempfile.TemporaryFile()
306 
307        chunksize = 1024*8
308        lastchunkreport= 0.0001
309
310        readb = 0
311        while 1:
312            buf = fdown.read(chunksize)
313            readb += len(buf)
314
315            while float(readb)/size > lastchunkreport+0.01:
316                #print float(readb)/size, lastchunkreport + 0.01, float(readb)/size - lastchunkreport
317                lastchunkreport += 0.01
318                if callback:
319                    callback()
320            if not buf:
321                break
322            f.write(buf)
323
324        fdown.close()
325        f.seek(0)
326
327        shutil.copyfileobj(f, open(target, "wb"))
328
329        if callback:
330            callback()
331
332    def _searchinfo(self):
333        domains = self.listdomains()
334        infos = {}
335        for dom in domains:
336            dominfo = self.allinfo(dom)
337            for a,b in dominfo.items():
338                infos[(dom, a)] = b
339        return infos
340
341    def search(self, sstrings, **kwargs):
342        """
343        Search for files on the repository where all substrings in a list
344        are contained in at least one choosen field (tag, title, name). Return
345        a list of tuples: first tuple element is the file's domain, second its
346        name. As for now the search is performed locally, therefore
347        information on files in repository is transfered on first call of
348        this function.
349        """
350        if not self.searchinfo:
351            self.searchinfo = self._searchinfo()
352        return _search(self.searchinfo, sstrings, **kwargs)
353
354    def info(self, domain, filename):
355        """Return a dictionary containing repository file info.
356        Keys: title, tags, size, datetime."""
357        return _parseFileInfo(self._open('info', { 'domain': domain, 'filename': filename }))
358
359    def downloadFH(self, domain, filename):
360        """Return a file handle to the file that we would like to download."""
361        return self._handle('download', { 'domain': domain, 'filename': filename })
362
363    def list(self, domain):
364        return _parseList(self._open('list', { 'domain': domain }))
365
366    def listdomains(self):
367        """List all domains on repository."""
368        return _parseList(self._open('listdomains', {}))
369
370    def allinfo(self, domain):
371        """Go through all accessible files in a given domain and return a
372        dictionary, where key is file's name and value its info.
373        """
374        return _parseAllFileInfo(self._open('allinfo', { 'domain': domain }))
375
376    def index(self):
377        return self._open('index', {})
378
379    def _authen(self):
380        """
381        Did the user choose authentication?
382        """
383        if self.username and self.password:
384            return True
385        else:
386            return False
387
388    def _server_request(self, root, command, data, repeat=2):
389        def do():
390            opener = self._getOpener()
391            #the next lines work for poster 0.6.0
392            #datagen, headers = poster.encode.multipart_encode(data)
393            #request = urllib2.Request(root+command, datagen, headers)
394
395            if data:
396                request = urllib2.Request(root+command, data)
397            else:
398                request = urllib2.Request(root+command)
399
400            #directy add authorization headers
401            if self._authen():
402                auth = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] 
403                request.add_header('Authorization', 'Basic %s' % auth ) # Add Auth header to request
404           
405            return opener.open(request)
406        if repeat <= 0:
407            return do()
408        else:
409            try:
410                return do()
411            except:
412                return self._server_request(root, command, data, repeat=repeat-1)
413   
414    def _handle(self, command, data):
415        data2 = self._addAccessCode(data)
416        addr = self.publicroot
417        if self._authen():
418            addr = self.secureroot
419        return self._server_request(addr, command, data)
420
421    def _open(self, command, data):
422        return self._handle(command, data).read()
423
424    def _addAccessCode(self, data):
425        if self.access_code != None:
426            data = data.copy()
427            data["access_code"] = self.access_code
428        return data
429
430def download(domain, filename, serverfiles=None, callback=None, 
431    extract=True, verbose=True):
432    """Downloads file from the repository to local orange installation.
433    To download files as an authenticated user you should also pass an
434    instance of ServerFiles class. Callback can be a function without
435    arguments. It will be called once for each downloaded percent of
436    file: 100 times for the whole file."""
437
438    if not serverfiles:
439        serverfiles = ServerFiles()
440
441    info = serverfiles.info(domain, filename)
442    specialtags = dict([tag.split(":") for tag in info["tags"] if tag.startswith("#") and ":" in tag])
443    extract = extract and ("#uncompressed" in specialtags or "#compression" in specialtags)
444    target = localpath(domain, filename)
445    callback = DownloadProgress(filename, int(info["size"])) if verbose and not callback else callback   
446    serverfiles.download(domain, filename, target + ".tmp" if extract else target, callback=callback)
447   
448    #file saved, now save info file
449
450    _save_file_info(target + '.info', info)
451   
452    if extract:
453        import tarfile, gzip, shutil
454        if specialtags.get("#compression") == "tar.gz" and specialtags.get("#files"):
455            f = tarfile.open(target + ".tmp")
456            f.extractall(localpath(domain))
457            shutil.copyfile(target + ".tmp", target)
458        if filename.endswith(".tar.gz"):
459            f = tarfile.open(target + ".tmp")
460            try:
461                os.mkdir(target)
462            except Exception:
463                pass
464            f.extractall(target)
465        elif specialtags.get("#compression") == "gz":
466            f = gzip.open(target + ".tmp")
467            shutil.copyfileobj(f, open(target, "wb"))
468        f.close()
469        os.remove(target + ".tmp")
470
471    if type(callback) == DownloadProgress:
472        callback.finish()
473
474def localpath_download(domain, filename, **kwargs):
475    """
476    Return local path for the given domain and file. If file does not exist,
477    download it. Additional arguments are passed to the :obj:`download` function.
478    """
479    pathname = localpath(domain, filename)
480    if not os.path.exists(pathname):
481        download(domain, filename, **kwargs)
482    return pathname
483
484def listfiles(domain):
485    """List all files from a domain in a local repository."""
486    dir = localpath(domain)
487    try:
488        files = [a for a in os.listdir(dir) if a[-5:] == '.info' ]
489    except:
490        files = []
491    okfiles = []
492
493    for file in files:
494        #if file to exists without info
495        if os.path.exists(os.path.join(dir,file[:-5])):
496            #check info format - needs to be valid
497            try:
498                _open_file_info(os.path.join(dir,file))
499                okfiles.append(file[:-5])
500            except:
501                pass
502
503    return okfiles
504
505def remove(domain, filename):
506    """Remove a file from local repository."""
507    filename = localpath(domain, filename)
508    import shutil
509   
510    specialtags = dict([tag.split(":") for tag in info(domain, filename)["tags"] if tag.startswith("#") and ":" in tag])
511    todelete = [filename, filename + ".info"] 
512    if "#files" in specialtags:
513        todelete.extend([os.path.join(localpath(domain), path) for path in specialtags.get("#files").split("!@")])
514#    print todelete
515    for path in todelete:
516        try:
517            if os.path.isdir(path):
518                shutil.rmtree(path)
519            elif os.path.isfile(path):
520                os.remove(path)
521        except OSError, ex:
522            print "Failed to delete", path, "due to:", ex
523   
524def remove_domain(domain, force=False):
525    """Remove a domain. If force is True, domain is removed even
526    if it is not empty (contains files)."""
527    directory = localpath(domain)
528    if force:
529        import shutil
530        shutil.rmtree(directory)
531    else:
532        os.rmdir(directory)
533
534def listdomains():
535    """List all file domains in the local repository."""
536    dir = localpath()
537    _create_path(dir)
538    files = [ a for a in os.listdir(dir) ]
539    ok = []
540    for file in files:
541        if os.path.isdir(os.path.join(dir, file)):
542            ok.append(file)
543    return ok
544
545def info(domain, filename):
546    """Returns info of a file in a local repository."""
547    target = localpath(domain, filename)
548    return _open_file_info(target + '.info')
549
550def allinfo(domain):
551    """Goes through all files in a domain on a local repository and returns a
552    dictionary, where keys are names of the files and values are their
553    information."""
554    files = listfiles(domain)
555    dic = {}
556    for filename in files:
557        target = localpath(domain, filename)
558        dic[filename] = info(domain, target)
559    return dic
560
561def needs_update(domain, filename, access_code=None):
562    """True if a file does not exist in the local repository
563    or if there is a newer version on the server."""
564    if filename not in listfiles(domain):
565        return True
566    dt_fmt = "%Y-%m-%d %H:%M:%S"
567    dt_local = datetime.datetime.strptime(
568        info(domain, filename)["datetime"][:19], dt_fmt)
569    server = ServerFiles(access_code=access_code)
570    dt_server = datetime.datetime.strptime(
571        server.info(domain, filename)["datetime"][:19], dt_fmt)
572    return dt_server > dt_local
573
574def update(domain, filename, access_code=None, verbose=True):
575    """Downloads the corresponding file from the server and places it in
576    the local repository, but only if the server copy of the file is newer
577    or the local copy does not exist. An optional access_code can be given
578    to allow server access if the file is protected.
579    """
580    if needs_update(domain, filename, access_code=access_code):
581        if not access_code:
582            download(domain, filename, verbose=verbose)
583        else:
584            server = orngServerFiles.ServerFiles(access_code=access_code)
585            download(domain, filename, serverfiles=server)
586       
587def _searchinfo():
588    domains = listdomains()
589    infos = {}
590    for dom in domains:
591        dominfo = allinfo(dom)
592        for a,b in dominfo.items():
593            infos[(dom, a)] = b
594    return infos
595
596def _search(si, sstrings, caseSensitive=False, inTag=True, inTitle=True, inName=True):
597    """
598    sstrings contain a list of search strings
599    """
600    found = []
601
602    for (dom,fn),info in si.items():
603        target = ""
604        if inTag: target += " ".join(info['tags'])
605        if inTitle: target += info['title']
606        if inName: target += fn
607        if not caseSensitive: target = target.lower()
608
609        match = True
610        for s in sstrings:
611            if not caseSensitive:
612                s = s.lower()
613            if s not in target:
614                match= False
615                break
616               
617        if match:
618            found.append((dom,fn))   
619       
620    return found
621
622def search(sstrings, **kwargs):
623    """Search for files in the local repository where all substrings in a list
624    are contained in at least one chosen field (tag, title, name). Return a
625    list of tuples: first tuple element is the domain of the file, second
626    its name."""
627    si = _searchinfo()
628    return _search(si, sstrings, **kwargs)
629
630class DownloadProgress(ConsoleProgressBar):
631    redirect = None
632    lock = threading.RLock()
633    def sizeof_fmt(num):
634        for x in ['bytes','KB','MB','GB','TB']:
635            if num < 1024.0:
636                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
637            num /= 1024.0
638           
639    def __init__(self, filename, size):
640        print "Downloading", filename
641        ConsoleProgressBar.__init__(self, "progress:", 20)
642        self.size = size
643        self.starttime = time.time()
644        self.speed = 0.0
645
646    def sizeof_fmt(self, num):
647        for x in ['bytes','KB','MB','GB','TB']:
648            if num < 1024.0:
649                return "%3.1f %s" % (num, x) if x <> 'bytes' else "%1.0f %s" % (num, x)
650            num /= 1024.0
651
652    def getstring(self):
653        speed = int(self.state * self.size / 100.0 / (time.time() - self.starttime))
654        eta = (100 - self.state) * self.size / 100.0 / speed
655        return ConsoleProgressBar.getstring(self) + %s  %12s/s  %3i:%02i ETA" % (self.sizeof_fmt(self.size), self.sizeof_fmt(speed), eta/60, eta%60)
656       
657    def __call__(self, *args, **kwargs):
658        ret = ConsoleProgressBar.__call__(self, *args, **kwargs)
659        if self.redirect:
660            self.redirect(self.state)
661        return ret
662   
663    class RedirectContext(object):
664        def __enter__(self):
665            DownloadProgress.lock.acquire()
666            return DownloadProgress
667       
668        def __exit__(self, ex_type, value, tb):
669            DownloadProgress.redirect = None
670            DownloadProgress.lock.release()
671            return False
672       
673    @classmethod
674    def setredirect(cls, redirect):
675        cls.redirect = staticmethod(redirect)
676        return cls.RedirectContext()
677   
678    @classmethod
679    def __enter__(cls):
680        cls.lock.acquire()
681        return cls
682   
683    @classmethod
684    def __exit__(cls, exc_type, exc_value, traceback):
685        cls.lock.release()
686        return False
687
688def consoleupdate(domains=None, searchstr="essential"):
689    domains = domains or listdomains()
690    sf = ServerFiles()
691    info = dict((d, sf.allinfo(d)) for d in domains)
692    def searchmenu():
693        def printmenu():
694            print "\tSearch tags:", search
695            print "\t1. Add tag."
696            print "\t2. Clear tags."
697            print "\t0. Return to main menu."
698            return raw_input("\tSelect option:")
699        search = searchstr
700        while True:
701            response = printmenu().strip()
702            if response == "1":
703                search += " " + raw_input("\tType new tag/tags:")
704            elif response == "2":
705                search = ""
706            elif response == "0":
707                break
708            else:
709                print "\tUnknown option!"
710        return search
711
712    def filemenu(searchstr=""):
713        files = [None]
714        for i, (dom, file) in enumerate(sf.search(searchstr.split())):
715            print "\t%i." % (i + 1), info[dom][file]["title"]
716            files.append((dom, file))
717        print "\t0. Return to main menu."
718        print "\tAction: d-download (e.g. 'd 1' downloads first file)"
719        while True:
720            response = raw_input("\tAction:").strip()
721            if response == "0":
722                break
723            try:
724                action, num = response.split(None, 1)
725                num = int(num)
726            except Exception, ex:
727                print "Unknown option!"
728                continue
729            try:
730                if action.lower() == "d":
731                    download(*(files[num]))
732                    print "\tSuccsessfully downloaded", files[num][-1]
733            except Exception, ex:
734                print "Error occured!", ex
735
736    def printmenu():
737        print "Update database main menu:"
738        print "1. Enter search tags (refine search)."
739        print "2. Print matching available files."
740        print "3. Print all available files."
741        print "4. Update all local files."
742        print "0. Exit."
743        return raw_input("Select option:")
744   
745    while True:
746        try:
747            response = printmenu().strip()
748            if response == "1":
749                searchstr = searchmenu()
750            elif response == "2":
751                filemenu(searchstr)
752            elif response == "3":
753                filemenu("")
754            elif response == "4":
755                update_local_files()
756            elif response == "0":
757                break
758            else:
759                print "Unknown option!"
760        except Exception, ex:
761            print "Error occured:", ex
762
763def update_local_files(verbose=True):
764    sf = ServerFiles()
765    for domain, filename in search(""):
766        uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
767        if not uptodate:
768            download(domain, filename, sf)
769        if verbose:
770            print filename, "Ok" if uptodate else "Updated"
771
772def update_by_tags(tags=["essential"], domains=[], verbose=True):
773    sf = ServerFiles()
774    for domain, filename in sf.search(tags + domains, inTitle=False, inName=False):
775        if domains and domain not in domain:
776            continue
777        if os.path.exists(localpath(domain, filename)+".info"):
778            uptodate = sf.info(domain, filename)["datetime"] <= info(domain, filename)["datetime"]
779        else:
780            uptodate = False
781        if not uptodate:
782            download(domain, filename, sf)
783        if verbose:
784            print filename, "Ok" if uptodate else "Updated"
785           
786def _example(myusername, mypassword):
787
788    locallist = listfiles('test')
789    for l in locallist:
790        print info('test', l)
791
792    s = ServerFiles()
793
794    print "testing connection - public"
795    print "AN", s.index()
796
797    #login as an authenticated user
798    s = ServerFiles(username=myusername, password=mypassword)
799   
800    """
801    print "Server search 1"
802    import time
803    t = time.time()
804    print s.search(["rat"])
805    print time.time() - t
806
807    t = time.time()
808    print s.search(["human", "ke"])
809    print time.time() - t
810    """
811
812    print "testing connection - private"
813    print "AN", s.index()
814
815    #create domain
816    try: 
817        s.create_domain("test") 
818    except:
819        print "Failed to create the domain"
820        pass
821
822    files = s.listfiles('test')
823    print "Files in test", files
824
825    print "uploading"
826
827    #upload this file - save it by a different name
828    s.upload('test', 'osf-test.py', 'serverfiles.py', title="NT", tags=["fkdl","fdl"])
829    #make it public
830    s.unprotect('test', 'osf-test.py')
831
832    #login anonymously
833    s = ServerFiles()
834
835    #list files in the domain "test"
836    files = s.listfiles('test')
837    print "ALL FILES:", files
838
839    for f in files:
840        fi = s.info('test', f) 
841        print "--------------------------------------", f
842        print "INFO", fi
843        print s.downloadFH('test', f).read()[:100] #show first 100 characters
844        print "--------------------------------------"
845
846    #login as an authenticated user
847    s = ServerFiles(username=myusername, password=mypassword)
848
849    print s.listdomains()
850
851    s.remove('test', 'osf-test.py')
852
853    s = ServerFiles()
854
855    print s.listdomains()
856
857
858if __name__ == '__main__':
859    _example(sys.argv[1], sys.argv[2])
Note: See TracBrowser for help on using the repository browser.