# Source code for sagenb.notebook.notebook

    # -*- coding: utf-8 -*
    """
    The Sage Notebook

    AUTHORS:

    - William Stein
    """

    #############################################################################
    #
    #       Copyright (C) 2006-2009 William Stein <wstein@gmail.com>
    #  Distributed under the terms of the GNU General Public License (GPL)
    #  The full text of the GPL is available at:
    #                  http://www.gnu.org/licenses/
    #
    #############################################################################
    from __future__ import print_function, absolute_import

    # For debugging sometimes it is handy to use only the reference implementation.
    USE_REFERENCE_WORKSHEET_PROCESSES = False

    # System libraries
    import os
    import random
    import re
    import shutil
    import socket
    import time
    import bz2
    from cgi import escape

    try:
        import cPickle as pickle
    except ImportError:
        import pickle

    try:
        basestring
    except NameError:
        basestring = (str, bytes)

    from six import iteritems

    # Sage libraries
    from sagenb.misc.misc import (pad_zeros, cputime, tmp_dir, load, save,
                                  ignore_nonexistent_files, unicode_str)

    # Sage Notebook
    from . import css          # style
    from . import js           # javascript
    from . import worksheet    # individual worksheets (which make up a notebook)
    from . import config       # internal configuration stuff (currently, just keycodes)
    from . import keyboards    # keyboard layouts
    from . import server_conf  # server configuration
    from . import user_conf    # user configuration
    from . import user         # users
    from   .template import template, prettify_time_ago
    from flask_babel import gettext, lazy_gettext

    try:
        # sage is installed
        import sage
        # [(string: name, bool: optional)]
        SYSTEMS = [('sage', False),
                   ('gap', False),
                   ('gp', False),
                   ('html', False),
                   ('latex', False),
                   ('maxima', False),
                   ('python', False),
                   ('r', False),
                   ('sh', False),
                   ('singular', False),
                   ('axiom', True),
                   ('fricas', True),
                   ('kash', True),
                   ('macaulay2', True),
                   ('magma', True),
                   ('maple', True,),
                   ('mathematica', True),
                   ('matlab', True),
                   ('mupad', True),
                   ('octave', True),
                   ('scilab', True)]
    except ImportError:
        # sage is not installed
        SYSTEMS = [('sage', True)]    # but gracefully degenerated version of sage mode, e.g., preparsing is trivial

    # We also record the system names without (optional) since they are
    # used in some of the html menus, etc.
    SYSTEM_NAMES = [v[0] for v in SYSTEMS]

    MATHJAX = True

    JEDITABLE_TINYMCE  = True

    class WorksheetDict(dict):
        def __init__(self, notebook, *args, **kwds):
            self.notebook = notebook
            self.storage = notebook._Notebook__storage
            dict.__init__(self, *args, **kwds)

        def __getitem__(self, item):
            if item in self:
                return dict.__getitem__(self, item)

            try:
                if '/' not in item:
                    raise KeyError(item)
            except TypeError:
                raise KeyError(item)

            username, id = item.split('/')
            try:
                id=int(id)
            except ValueError:
                raise KeyError(item)
            try:
                worksheet = self.storage.load_worksheet(username, id)
            except ValueError:
                raise KeyError(item)

            dict.__setitem__(self, item, worksheet)
            return worksheet

    class Notebook(object):
        HISTORY_MAX_OUTPUT = 92*5
        HISTORY_NCOLS = 90

        def __init__(self, dir, user_manager = None):

            if isinstance(dir, basestring) and len(dir) > 0 and dir[-1] == "/":
                dir = dir[:-1]

            if not dir.endswith('.sagenb'):
                raise ValueError("dir (=%s) must end with '.sagenb'" % dir)

            self._dir = dir

            # For now we only support the FilesystemDatastore storage
            # backend.
            from sagenb.storage import FilesystemDatastore
            S = FilesystemDatastore(dir)
            self.__storage = S

            # Now set the configuration, loaded from the datastore.
            try:
                self.__conf = S.load_server_conf()
            except IOError:
                # Worksheet has never been saved before, so the server conf doesn't exist.
                self.__worksheets = WorksheetDict(self)

            from sagenb.notebook.user_manager import SimpleUserManager, OpenIDUserManager
            self._user_manager = OpenIDUserManager(conf=self.conf()) if user_manager is None else user_manager

            # Set up email notification logger
            import logging
            from sagenb.notebook.notification import logger, TwistedEmailHandler
            logger.addHandler(TwistedEmailHandler(self.conf(), logging.ERROR))
            # also log to stderr
            logger.addHandler(logging.StreamHandler())

            # Set the list of users
            try:
                S.load_users(self._user_manager)
            except IOError:
                pass

            # Set the list of worksheets
            W = WorksheetDict(self)
            self.__worksheets = W

            # Store / Refresh public worksheets
            for id_number in os.listdir(self.__storage._abspath(self.__storage._user_path("pub"))):
                if id_number.isdigit():
                    a = "pub/"+str(id_number)
                    if a not in self.__worksheets:
                        try:
                            self.__worksheets[a] = self.__storage.load_worksheet("pub",int(id_number))
                        except Exception:
                            import traceback
                            print("Warning: problem loading %s/%s: %s" % ("pub", int(id_number), traceback.format_exc()))

            # Set the openid-user dict
            try:
                self._user_manager.load(S)
            except IOError:
                pass

        def delete(self):
            """
            Delete all files related to this notebook.

            This is used for doctesting mainly. This command is obviously
            *VERY* dangerous to use on a notebook you actually care about.
            You could easily lose all data.

            EXAMPLES::

                sage: tmp = tmp_dir(ext='.sagenb')
                sage: nb = sagenb.notebook.notebook.Notebook(tmp)
                sage: sorted(os.listdir(tmp))
                ['home']
                sage: nb.delete()

            Now the directory is gone.::

                sage: os.listdir(tmp)
                Traceback (most recent call last):
                ...
                OSError: [Errno 2] No such file or directory: '...
            """
            self.__storage.delete()

        def systems(self, username=None):
            systems = []
            for system in SYSTEMS:
                if system[1]:
                    systems.append(system[0] + ' (' + lazy_gettext('optional') + ')')
                else:
                    systems.append(system[0])
            return systems

        def system_names(self):
            return SYSTEM_NAMES

        def user_manager(self):
            """
            Returns self's UserManager object.

            EXAMPLES::

                sage: n = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: n.user_manager()
                <sagenb.notebook.user_manager.OpenIDUserManager object at 0x...>
            """
            return self._user_manager

        ##########################################################
        # Users
        ##########################################################
        def create_default_users(self, passwd):
            """
            Create the default users for a notebook.

            INPUT:

            - ``passwd`` - a string

            EXAMPLES::

                sage: from six import iteritems
                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: sorted(list(iteritems(nb.user_manager().users())))
                [('_sage_', _sage_), ('admin', admin), ('guest', guest), ('pub', pub)]
                sage: sorted(list(iteritems(nb.user_manager().passwords()))) #random
                [('_sage_', ''), ('admin', ''), ('guest', ''), ('pub', '')]
                sage: nb.create_default_users('newpassword')
                WARNING: User 'pub' already exists -- and is now being replaced.
                WARNING: User '_sage_' already exists -- and is now being replaced.
                WARNING: User 'guest' already exists -- and is now being replaced.
                WARNING: User 'admin' already exists -- and is now being replaced.
                sage: sorted(list(iteritems(nb.user_manager().passwords()))) #random
                [('_sage_', ''), ('admin', ''), ('guest', ''), ('pub', '')]
                sage: len(list(iteritems(nb.user_manager().passwords())))
                4
            """
            self.user_manager().create_default_users(passwd)

        def user(self, username):
            """
            Return an instance of the User class given the ``username`` of a user
            in a notebook.

            INPUT:

            - ``username`` - a string

            OUTPUT:

            - an instance of User

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.user_manager().create_default_users('password')
                sage: nb.user('admin')
                admin
                sage: nb.user('admin').get_email()
                ''
                sage: nb.user('admin').password() #random
                '256$7998210096323979f76e9fedaf1f85bda1561c479ae732f9c1f1abab1291b0b9$373f16b9d5fab80b9a9012af26a6b2d52d92b6d4b64c1836562cbd4264a6e704'
            """
            return self.user_manager().user(username)

        def valid_login_names(self):
            """
            Return a list of users that can log in.

            OUTPUT:

            - a list of strings

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: nb.valid_login_names()
                ['admin']
                sage: nb.user_manager().add_user('Mark', 'password', '', force=True)
                sage: nb.user_manager().add_user('Sarah', 'password', '', force=True)
                sage: nb.user_manager().add_user('David', 'password', '', force=True)
                sage: sorted(nb.valid_login_names())
                ['David', 'Mark', 'Sarah', 'admin']
            """
            return self.user_manager().valid_login_names()

        def readonly_user(self, username):
            """
            Returns True if the user is supposed to only be a read-only user.
            """
            return self.__storage.readonly_user(username)

        ##########################################################
        # Publishing worksheets
        ##########################################################
        def _initialize_worksheet(self, src, W):
            r"""
            Initialize a new worksheet from a source worksheet.

            INPUT:

            - ``src`` - a Worksheet instance; the source

            - ``W`` - a new Worksheet instance; the target
            """
            # Note: Each Worksheet method *_directory actually creates a
            # directory, if it doesn't already exist.

            # More compact, but significantly less efficient?
            #      shutil.rmtree(W.cells_directory(), ignore_errors=True)
            #      shutil.rmtree(W.data_directory(), ignore_errors=True)
            #      shutil.rmtree(W.snapshots_directory(), ignore_errors=True)
            #      shutil.copytree(src.cells_directory(), W.cells_directory())
            #      shutil.copytree(src.data_directory(), W.data_directory())

            for sub in ['cells', 'data', 'snapshots']:
                target_dir = os.path.join(W.directory(), sub)
                if os.path.exists(target_dir):
                    shutil.rmtree(target_dir, ignore_errors=True)

            # Copy images, data files, etc.
            for sub in ['cells', 'data']:
                source_dir = os.path.join(src.directory(), sub)
                if os.path.exists(source_dir):
                    target_dir = os.path.join(W.directory(), sub)
                    shutil.copytree(source_dir, target_dir)

            W.edit_save(src.edit_text())
            W.save()

        def pub_worksheets(self):
            path = self.__storage._abspath(self.__storage._user_path("pub"))
            v = []
            a = ""
            for id_number in os.listdir(path):
                if id_number.isdigit():
                    a = "pub"+"/"+id_number
                    if a in self.__worksheets:
                        v.append(self.__worksheets[a])
                    else:
                        try:
                            self.__worksheets[a] = self.__storage.load_worksheet("pub", int(id_number))
                            v.append(self.__worksheets[a])
                        except Exception:
                            import traceback
                            print("Warning: problem loading %s/%s: %s" % ("pub", id_number, traceback.format_exc()))
            return v

        def users_worksheets(self, username):
            r"""
            Returns all worksheets owned by `username`
            """

            if username == "pub":
                return self.pub_worksheets()

            worksheets = self.__storage.worksheets(username)
            # if a worksheet has already been loaded in self.__worksheets, return
            # that instead since worksheets that are already running should be
            # noted as such
            return [self.__worksheets[w.filename()] if w.filename() in self.__worksheets else w for w in worksheets]

        def users_worksheets_view(self, username):
            r"""
            Returns all worksheets viewable by `username`
            """
            # Should return worksheets from self.__worksheets if possible
            worksheets = self.users_worksheets(username)
            user=self.user_manager().user(username)
            viewable_worksheets=[self.__storage.load_worksheet(owner, id) for owner,id in user.viewable_worksheets()]
            # we double-check that we can actually view these worksheets
            # just in case someone forgets to update the map
            worksheets.extend([w for w in viewable_worksheets if w.is_viewer(username)])
            # if a worksheet has already been loaded in self.__worksheets, return that instead
            # since worksheets that are already running should be noted as such
            return [self.__worksheets[w.filename()] if w.filename() in self.__worksheets else w for w in worksheets]

        def publish_worksheet(self, worksheet, username):
            r"""
            Publish a user's worksheet.  This creates a new worksheet in
            the 'pub' directory with the same contents as ``worksheet``.

            INPUT:

            - ``worksheet`` - an instance of Worksheet

            - ``username`` - a string

            OUTPUT:

            - a new or existing published instance of Worksheet

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.load_notebook(tmp_dir(ext='.sagenb'))
                sage: nb.user_manager().add_user('Mark','password','',force=True)
                sage: W = nb.new_worksheet_with_title_from_text('First steps', owner='Mark')
                sage: nb.worksheet_names()
                ['Mark/0']
                sage: nb.create_default_users('password')
                sage: nb.publish_worksheet(nb.get_worksheet_with_filename('Mark/0'), 'Mark')
                pub/0: [Cell 1: in=, out=]
                sage: sorted(nb.worksheet_names())
                ['Mark/0', 'pub/0']
            """
            W = None

            # Reuse an existing published version
            for X in self.get_worksheets_with_owner('pub'):
                if (X.worksheet_that_was_published() == worksheet):
                    W = X

            # Or create a new one.
            if W is None:
                W = self.create_new_worksheet(worksheet.name(), 'pub')

            # Copy cells, output, data, etc.
            self._initialize_worksheet(worksheet, W)

            # Update metadata.
            W.set_worksheet_that_was_published(worksheet)
            W.move_to_archive(username)
            worksheet.set_published_version(W.filename())
            W.record_edit(username)
            W.set_name(worksheet.name())
            self.__worksheets[W.filename()] = W
            W.save()
            return W

        ##########################################################
        # Moving, copying, creating, renaming, and listing worksheets
        ##########################################################

        def scratch_worksheet(self):
            try:
                return self.__scratch_worksheet
            except AttributeError:
                W = self.create_new_worksheet('scratch', '_sage_')
                self.__scratch_worksheet = W
                return W

        def create_new_worksheet(self, worksheet_name, username):
            if username!='pub' and self.user_manager().user_is_guest(username):
                raise ValueError("guests cannot create new worksheets")

            W = self.worksheet(username)

            W.set_system(self.system(username))
            W.set_name(worksheet_name)
            self.save_worksheet(W)
            self.__worksheets[W.filename()] = W

            return W

        def copy_worksheet(self, ws, owner):
            W = self.create_new_worksheet('default', owner)
            self._initialize_worksheet(ws, W)
            name = "Copy of %s" % ws.name()
            W.set_name(name)
            return W

        def delete_worksheet(self, filename):
            """
            Delete the given worksheet and remove its name from the worksheet
            list.  Raise a KeyError, if it is missing.

            INPUT:

            - ``filename`` - a string
            """
            try:
                W = self.__worksheets[filename]
            except KeyError:
                raise KeyError("Attempt to delete missing worksheet '%s'" % filename)

            W.quit()
            shutil.rmtree(W.directory(), ignore_errors=False)
            self.deleted_worksheets()[filename] = W

        def deleted_worksheets(self):
            try:
                return self.__deleted_worksheets
            except AttributeError:
                self.__deleted_worksheets = {}
                return self.__deleted_worksheets

        def empty_trash(self, username):
            """
            Empty the trash for the given user.

            INPUT:

            -  ``username`` - a string

            This empties the trash for the given user and cleans up all files
            associated with the worksheets that are in the trash.

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.user_manager().add_user('sage','sage','sage@sagemath.org',force=True)
                sage: W = nb.new_worksheet_with_title_from_text('Sage', owner='sage')
                sage: W._notebook = nb
                sage: W.move_to_trash('sage')
                sage: nb.worksheet_names()
                ['sage/0']
                sage: nb.empty_trash('sage')
                sage: nb.worksheet_names()
                []
            """
            X = self.get_worksheets_with_viewer(username)
            X = [W for W in X if W.is_trashed(username)]
            for W in X:
                W.delete_user(username)
                if W.owner() is None:
                    self.delete_worksheet(W.filename())

        def worksheet_names(self):
            """
            Return a list of all the names of worksheets in this notebook.

            OUTPUT:

            - a list of strings.

            EXAMPLES:

            We make a new notebook with two users and two worksheets,
            then list their names::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.user_manager().add_user('sage','sage','sage@sagemath.org',force=True)
                sage: W = nb.new_worksheet_with_title_from_text('Sage', owner='sage')
                sage: nb.user_manager().add_user('wstein','sage','wstein@sagemath.org',force=True)
                sage: W2 = nb.new_worksheet_with_title_from_text('Elliptic Curves', owner='wstein')
                sage: nb.worksheet_names()
                ['sage/0', 'wstein/0']
            """
            W = self.__worksheets.keys()
            W.sort()
            return W

        ##########################################################
        # Information about the pool of worksheet compute servers
        ##########################################################

        def server_pool(self):
            return self.conf()['server_pool']

        def set_server_pool(self, servers):
            self.conf()['server_pool'] = servers

        def get_ulimit(self):
            try:
                return self.__ulimit
            except AttributeError:
                self.__ulimit = ''
                return ''

        def set_ulimit(self, ulimit):
            self.__ulimit = ulimit

        def get_server(self):
            P = self.server_pool()
            if P is None or len(P) == 0:
                return None
            try:
                self.__server_number = (self.__server_number + 1) % len(P)
                i = self.__server_number
            except AttributeError:
                self.__server_number = 0
                i = 0
            return P[i]

        def new_worksheet_process(self):
            """
            Return a new worksheet process object with parameters determined by
            configuration of this notebook server.
            """
            from sagenb.interfaces import (WorksheetProcess_ExpectImplementation,
                                           WorksheetProcess_ReferenceImplementation,
                                           WorksheetProcess_RemoteExpectImplementation)

            if USE_REFERENCE_WORKSHEET_PROCESSES:
                return WorksheetProcess_ReferenceImplementation()

            ulimit = self.get_ulimit()
            from sagenb.interfaces import ProcessLimits
            # We have to parse the ulimit format to our ProcessLimits.
            # The typical format is.
            # '-u 400 -v 1000000 -t 3600'
            # Despite -t being cputime for ulimit, we map it to walltime,
            # since that is the only thing that really makes sense for a
            # notebook server.
            #    -u --> max_processes
            #    -v --> max_vmem (but we divide by 1000)
            #    -t -- > max_walltime

            max_vmem = max_cputime = max_walltime = None
            tbl = {'v': None, 'u': None, 't': None}
            for x in ulimit.split('-'):
                for k in tbl.keys():
                    if x.startswith(k):
                        tbl[k] = int(x.split()[1].strip())
            if tbl['v'] is not None:
                tbl['v'] = tbl['v'] / 1000.0

            process_limits = ProcessLimits(max_vmem=tbl['v'], max_walltime=tbl['t'],
                                           max_processes=tbl['u'])

            server_pool = self.server_pool()
            if not server_pool or len(server_pool) == 0:
                return WorksheetProcess_ExpectImplementation(process_limits=process_limits)
            else:
                import random
                user_at_host = random.choice(server_pool)
                python_command = os.path.join(os.environ['SAGE_ROOT'], 'sage -python')
                return WorksheetProcess_RemoteExpectImplementation(user_at_host=user_at_host,
                                 process_limits=process_limits,
                                 remote_python=python_command)

        def _python_command(self):
            """
            """
            try:
                return self.__python_command
            except AttributeError:
                pass

        ##########################################################
        # Configuration settings.
        ##########################################################
        def system(self, username=None):
            """
            The default math software system for new worksheets for a
            given user or the whole notebook (if username is None).
            """
            return self.user(username).conf()['default_system']

        def pretty_print(self, username=None):
            """
            The default typeset setting for new worksheets for
            a given user or the whole notebook (if username is None).

            TODO -- only implemented for the notebook right now
            """
            return self.user(username).conf()['default_pretty_print']

        def set_pretty_print(self, pretty_print):
            self.__pretty_print = pretty_print

        def color(self):
            """
            The default color scheme for the notebook.
            """
            try:
                return self.__color
            except AttributeError:
                self.__color = 'default'
                return self.__color

        def set_color(self, color):
            self.__color = color

        ##########################################################
        # The notebook history.
        ##########################################################
        def user_history(self, username):
            if not hasattr(self, '_user_history'):
                self._user_history = {}
            if username in self._user_history:
                return self._user_history[username]
            history = []
            for hunk in self.__storage.load_user_history(username):
                hunk = unicode_str(hunk)
                history.append(hunk)
            self._user_history[username] = history
            return history

        def create_new_worksheet_from_history(self, name, username, maxlen=None):
            W = self.create_new_worksheet(name, username)
            W.edit_save('Log Worksheet\n' + self.user_history_text(username, maxlen=None))
            return W

        def user_history_text(self, username, maxlen=None):
            history = self.user_history(username)
            if maxlen:
                history = history[-maxlen:]
            return '\n\n'.join([hunk.strip() for hunk in history])

        def add_to_user_history(self, entry, username):
            history = self.user_history(username)
            history.append(entry)
            maxlen = self.user_manager().user_conf(username)['max_history_length']
            while len(history) > maxlen:
                del history[0]

        ##########################################################
        # Importing and exporting worksheets to files
        ##########################################################
        def export_worksheet(self, worksheet_filename, output_filename, title=None):
            """
            Export a worksheet, creating a sws file on the file system.

            INPUT:

                -  ``worksheet_filename`` - a string e.g., 'username/id_number'

                -  ``output_filename`` - a string, e.g., 'worksheet.sws'

                - ``title`` - title to use for the exported worksheet (if
                   None, just use current title)
            """
            S = self.__storage
            W = self.get_worksheet_with_filename(worksheet_filename)
            S.save_worksheet(W)
            username = W.owner()
            id_number = W.id_number()
            S.export_worksheet(username, id_number, output_filename, title=title)

        def worksheet(self, username, id_number=None):
            """
            Create a new worksheet with given id_number belonging to the
            user with given username, or return an already existing
            worksheet.  If id_number is None, creates a new worksheet
            using the next available new id_number for the given user.

            INPUT:

                - ``username`` -- string

                - ``id_number`` - nonnegative integer or None (default)
            """
            S = self.__storage
            if id_number is None:
                id_number = self.new_id_number(username)
            try:
                W = S.load_worksheet(username, id_number)
            except ValueError:
                W = S.create_worksheet(username, id_number)
            self.__worksheets[W.filename()] = W
            return W

        def new_id_number(self, username):
            """
            Find the next worksheet id for the given user.
            """
            u = self.user(username).conf()
            id_number = u['next_worksheet_id_number']
            if id_number == -1:  # need to initialize
                id_number = max([w.id_number() for w in self.worksheet_list_for_user(username)] + [-1]) + 1
            u['next_worksheet_id_number'] = id_number + 1
            return id_number

        def new_worksheet_with_title_from_text(self, text, owner):
            name, _ = worksheet.extract_name(text)
            W = self.create_new_worksheet(name, owner)
            return W

        def change_worksheet_key(self, old_key, new_key):
            ws = self.__worksheets
            W = ws[old_key]
            ws[new_key] = W
            del ws[old_key]

        def import_worksheet(self, filename, owner):
            r"""
            Import a worksheet with the given ``filename`` and set its
            ``owner``.  If the file extension is not recognized, raise a
            ValueError.

            INPUT:

            -  ``filename`` - a string

            -  ``owner`` - a string

            OUTPUT:

            -  ``worksheet`` - a newly created Worksheet instance

            EXAMPLES:

            We create a notebook and import a plain text worksheet
            into it.

            ::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: name = tmp_filename() + '.txt'
                sage: open(name,'w').write('foo\n{{{\n2+3\n}}}')
                sage: W = nb.import_worksheet(name, 'admin')

            W is our newly-created worksheet, with the 2+3 cell in it::

                sage: W.name()
                u'foo'
                sage: W.cell_list()
                [TextCell 0: foo, Cell 1: in=2+3, out=]
            """
            if not os.path.exists(filename):
                raise ValueError("no file %s" % filename)

            # Figure out the file extension
            ext = os.path.splitext(filename)[1]
            if ext.lower() == '.txt':
                # A plain text file with {{{'s that defines a worksheet (no graphics).
                W = self._import_worksheet_txt(filename, owner)
            elif ext.lower() == '.sws':
                # An sws file (really a tar.bz2) which defines a worksheet with graphics, etc.
                W = self._import_worksheet_sws(filename, owner)
            elif ext.lower() == '.html':
                # An html file, which should contain the static version of
                # a sage help page, as generated by Sphinx
                html = open(filename).read()

                cell_pattern = r"""{{{id=.*?///.*?}}}"""
                docutils_pattern = r"""<meta name="generator" content="Docutils \S+: http://docutils\.sourceforge\.net/" />"""
                sphinx_pattern = r"""Created using <a href="http://sphinx\.pocoo\.org/">Sphinx</a>"""

                if re.search(cell_pattern, html, re.DOTALL) is not None:
                    W = self._import_worksheet_txt(filename, owner)
                elif re.search(docutils_pattern, html) is not None:
                    W = self._import_worksheet_docutils_html(filename, owner)
                elif re.search(sphinx_pattern, html) is not None:
                    W = self._import_worksheet_html(filename, owner)
                else:
                    # Unrecognized html file
                    # We do the default behavior, i.e. we import as if it was generated
                    # by Sphinx web page
                    W = self._import_worksheet_html(filename, owner)
            elif ext.lower() == '.rst':
                # A ReStructuredText file
                W = self._import_worksheet_rst(filename, owner)
            else:
                # We only support txt, sws, html and rst files
                raise ValueError("unknown extension '%s'" % ext)
            self.__worksheets[W.filename()] = W
            return W

        def _import_worksheet_txt(self, filename, owner):
            r"""
            Import a plain text file as a new worksheet.

            INPUT:

            -  ``filename`` - a string; a filename that ends in .txt

            -  ``owner`` - a string; the imported worksheet's owner

            OUTPUT:

            -  a new instance of Worksheet

            EXAMPLES:

            We write a plain text worksheet to a file and import it
            using this function.::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: name = tmp_filename() + '.txt'
                sage: open(name,'w').write('foo\n{{{\na = 10\n}}}')
                sage: W = nb._import_worksheet_txt(name, 'admin'); W
                admin/0: [TextCell 0: foo, Cell 1: in=a = 10, out=]
            """
            # Open the worksheet txt file and load it in.
            worksheet_txt = open(filename).read()
            # Create a new worksheet with the right title and owner.
            worksheet = self.new_worksheet_with_title_from_text(worksheet_txt, owner)
            # Set the new worksheet to have the contents specified by that file.
            worksheet.edit_save(worksheet_txt)
            return worksheet

        def _import_worksheet_sws(self, filename, username):
            r"""
            Import an sws format worksheet into this notebook as a new
            worksheet.

            INPUT:

            - ``filename`` - a string; a filename that ends in .sws;
               internally it must be a tar'd bz2'd file.

            - ``username`` - a string

            OUTPUT:

            - a new Worksheet instance

            EXAMPLES:

            We create a notebook, then make a worksheet from a plain text
            file first.::

                sage: nb = sagenb.notebook.notebook.load_notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: name = tmp_filename() + '.txt'
                sage: open(name,'w').write('{{{id=0\n2+3\n}}}')
                sage: W = nb.import_worksheet(name, 'admin')
                sage: W.filename()
                'admin/0'
                sage: sorted([w.filename() for w in nb.get_all_worksheets()])
                ['admin/0']

            We then export the worksheet to an sws file.::

                sage: sws = os.path.join(tmp_dir(), 'tmp.sws')
                sage: nb.export_worksheet(W.filename(), sws)

            Now we import the sws.::

                sage: W = nb._import_worksheet_sws(sws, 'admin')
                sage: nb._Notebook__worksheets[W.filename()] = W

            Yes, it's there now (as a new worksheet)::

                sage: sorted([w.filename() for w in nb.get_all_worksheets()])
                ['admin/0', 'admin/1']
            """
            id_number = self.new_id_number(username)
            worksheet = self.__storage.import_worksheet(username, id_number, filename)

            # I'm not at all convinced this is a good idea, since we
            # support multiple worksheets with the same title very well
            # already.  So it's commented out.
            # self.change_worksheet_name_to_avoid_collision(worksheet)

            return worksheet

        def _import_worksheet_html(self, filename, owner):
            r"""
            Import a static html help page generated by Sphinx as a new
            worksheet.

            INPUT:

            -  ``filename`` - a string; a filename that ends in .html

            -  ``owner`` - a string; the imported worksheet's owner

            OUTPUT:

            -  a new instance of Worksheet

            EXAMPLES:

            We write a plain text worksheet to a file and import it
            using this function.::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: name = tmp_filename() + '.html'
                sage: fd = open(name,'w')
                sage: fd.write(''.join([
                ....: '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"\n',
                ....: '  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n',
                ....: '\n',
                ....: '<html xmlns="http://www.w3.org/1999/xhtml">\n',
                ....: '  <head>\n',
                ....: '   <title>Test notebook &mdash; test</title>\n',
                ....: ' </head>\n',
                ....: '  <body>\n',
                ....: '   <div class="document">\n',
                ....: '      <div class="documentwrapper">\n',
                ....: '        <div class="bodywrapper">\n',
                ....: '          <div class="body">\n',
                ....: '<p>Here are some computations:</p>\n',
                ....: '\n',
                ....: '<div class="highlight-python"><div class="highlight"><pre>\n',
                ....: '<span class="gp">sage',
                ....: ': </span><span class="mi">1</span><span class="o">+</span><span class="mi">1</span>\n',
                ....: '<span class="go">2</span>\n',
                ....: '</pre></div></div>\n',
                ....: '\n',
                ....: '</div></div></div></div>\n',
                ....: '</body></html>']))
                sage: fd.close()
                sage: W = nb._import_worksheet_html(name, 'admin')
                sage: W.name()
                u'Test notebook -- test'
                sage: W.owner()
                'admin'
                sage: W.cell_list()
                [TextCell 1: <div class="document">
                      <div class="documentwrapper">
                        <div class="bodywrapper">
                          <div class="body">
                <p>Here are some computations:</p>
                <BLANKLINE>
                <div class="highlight-python">, Cell 0: in=1+1, out=
                2, TextCell 2: </div>
                <BLANKLINE>
                </div></div></div></div>]
                sage: cell = W.cell_list()[1]
                sage: cell.input_text()
                u'1+1'
                sage: cell.output_text()
                u'<pre class="shrunk">2</pre>'
            """
            # Inspired from sagenb.notebook.twist.WorksheetFile.render
            doc_page_html = open(filename).read()
            from .docHTMLProcessor import SphinxHTMLProcessor
            # FIXME: does SphinxHTMLProcessor raise an appropriate message
            # if the html file does not contain a Sphinx HTML page?
            doc_page = SphinxHTMLProcessor().process_doc_html(doc_page_html)

            from .misc import extract_title
            title = extract_title(doc_page_html).replace('&mdash;','--')

            worksheet = self.create_new_worksheet(title, owner)
            worksheet.edit_save(doc_page)

            # FIXME: An extra compute cell is always added to the end.
            # Pop it off.
            cells = worksheet.cell_list()
            cells.pop()

            return worksheet

        def _import_worksheet_rst(self, filename, owner):
            r"""
            Import a ReStructuredText file as a new worksheet.

            INPUT:

            -  ``filename`` - a string; a filename that ends in .rst

            -  ``owner`` - a string; the imported worksheet's owner

            OUTPUT:

            -  a new instance of Worksheet

            EXAMPLES:

                sage: sprompt = 'sage' + ':'
                sage: rst = '\n'.join(['=============',
                ....:     'Test Notebook',
                ....:     '=============',
                ....:     '',
                ....:     'Let\'s do some computations::',
                ....:     '',
                ....:     '    %s 2+2' % sprompt,
                ....:     '    4',
                ....:     '',
                ....:     '::',
                ....:     '',
                ....:     '    %s x^2' % sprompt,
                ....:     '    x^2'])
                sage: name = tmp_filename() + '.rst'
                sage: fd = open(name,'w')
                sage: fd.write(rst)
                sage: fd.close()
                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb._import_worksheet_rst(name, 'admin')
                sage: W.name()
                u'Test Notebook'
                sage: W.owner()
                'admin'
                sage: W.cell_list()
                [TextCell 2: <h1 class="title">Test Notebook</h1>
                <BLANKLINE>
                <p>Let's do some computations:</p>, Cell 0: in=2+2, out=
                4, Cell 1: in=x^2, out=
                x^2]
                sage: cell = W.cell_list()[1]
                sage: cell.input_text()
                u'2+2'
                sage: cell.output_text()
                u'<pre class="shrunk">4</pre>'

            """
            rst = open(filename).read()

            # docutils removes the backslashes if they are not escaped This is
            # not practical because backslashes are almost never escaped in
            # Sage docstrings written in ReST.  So if the user wants the
            # backslashes to be escaped automatically, he adds the comment
            # ".. escape-backslashes" in the input file
            if re.search(r'^\.\.[ \t]+escape-backslashes', rst, re.MULTILINE) is not None:
                rst = rst.replace('\\','\\\\')

            # Do the translation rst -> html (using docutils)
            from docutils.core import publish_parts
            D = publish_parts(rst, writer_name='html')
            title = D['title']
            html = D['whole']

            # Do the translation html -> txt
            from .docHTMLProcessor import docutilsHTMLProcessor
            translator = docutilsHTMLProcessor()
            worksheet_txt = translator.process_doc_html(html)

            # Create worksheet
            worksheet = self.create_new_worksheet(title, owner)
            worksheet.edit_save(worksheet_txt)

            return worksheet

        def _import_worksheet_docutils_html(self, filename, owner):
            r"""
            Import a static html help page generated by docutils as a new
            worksheet.

            INPUT:

            -  ``filename`` - a string; a filename that ends in .html

            -  ``owner`` - a string; the imported worksheet's owner

            OUTPUT:

            -  a new instance of Worksheet

            EXAMPLES:

                sage: sprompt = 'sage' + ':'
                sage: rst = '\n'.join(['=============',
                ....:     'Test Notebook',
                ....:     '=============',
                ....:     '',
                ....:     'Let\'s do some computations::',
                ....:     '',
                ....:     '    %s 2+2' % sprompt,
                ....:     '    4',
                ....:     '',
                ....:     '::',
                ....:     '',
                ....:     '    %s x^2' % sprompt,
                ....:     '    x^2'])
                sage: from docutils.core import publish_string
                sage: html = publish_string(rst, writer_name='html')
                sage: name = tmp_filename() + '.html'
                sage: fd = open(name,'w')
                sage: fd.write(html)
                sage: fd.close()
                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb._import_worksheet_docutils_html(name, 'admin')
                sage: W.name()
                u'Test Notebook'
                sage: W.owner()
                'admin'
                sage: W.cell_list()
                [TextCell 2: <h1 class="title">Test Notebook</h1>
                <BLANKLINE>
                <p>Let's do some computations:</p>, Cell 0: in=2+2, out=
                4, Cell 1: in=x^2, out=
                x^2]
                sage: cell = W.cell_list()[1]
                sage: cell.input_text()
                u'2+2'
                sage: cell.output_text()
                u'<pre class="shrunk">4</pre>'

            """
            html = open(filename).read()

            # Do the translation html -> txt
            from .docHTMLProcessor import docutilsHTMLProcessor
            translator = docutilsHTMLProcessor()
            worksheet_txt = translator.process_doc_html(html)

            # Extract title
            from .worksheet import extract_name
            title, _ = extract_name(worksheet_txt)
            if title.startswith('<h1 class="title">'):
                title = title[18:]
            if title.endswith('</h1>'):
                title = title[:-5]

            # Create worksheet
            worksheet = self.create_new_worksheet(title, owner)
            worksheet.edit_save(worksheet_txt)

            return worksheet

        def change_worksheet_name_to_avoid_collision(self, worksheet):
            """
            Change the display name of the worksheet if there is already a
            worksheet with the same name as this one.
            """
            name = worksheet.name()
            display_names = [w.name() for w in self.get_worksheets_with_owner(worksheet.owner())]
            if name in display_names:
                j = name.rfind('(')
                if j != -1:
                    name = name[:j].rstrip()
                i = 2
                while name + " (%s)" % i in display_names:
                    i += 1
                name = name + " (%s)" % i
                worksheet.set_name(name)

        ##########################################################
        # Server configuration
        ##########################################################

        def conf(self):
            try:
                return self.__conf
            except AttributeError:
                C = server_conf.ServerConfiguration()
                # if we are newly creating a notebook, then we want to
                # have a default model version of 1, currently
                # we can't just set the default value in server_conf.py
                # to 1 since it would then be 1 for notebooks without the
                # model_version property
                # TODO: distinguish between a new server config default values
                #  and default values for missing properties
                C['model_version']=1
                self.__conf = C
                return C

        ##########################################################
        # Computing control
        ##########################################################
        def set_not_computing(self):
            # unpickled, no worksheets will think they are
            # being computed, since they clearly aren't (since
            # the server just started).
            for W in self.__worksheets.values():
                W.set_not_computing()

        def quit(self):
            for W in list(self.__worksheets.values()):
                W.quit()

        def update_worksheet_processes(self):
            worksheet.update_worksheets()

        def quit_idle_worksheet_processes(self):
            timeout = self.conf()['idle_timeout']
            doc_timeout = self.conf()['doc_timeout']

            for W in self.__worksheets.values():
                if W.compute_process_has_been_started():
                    if W.docbrowser():
                        W.quit_if_idle(doc_timeout)
                    else:
                        W.quit_if_idle(timeout)

        def quit_worksheet(self, W):
            try:
                del self.__worksheets[W.filename()]
            except KeyError:
                pass

        ##########################################################
        # Worksheet HTML generation
        ##########################################################
        def worksheet_list_for_public(self, username, sort='last_edited', reverse=False, search=None):
            W = self.users_worksheets('pub')

            if search:
                W = [x for x in W if x.satisfies_search(search)]

            sort_worksheet_list(W, sort, reverse)  # changed W in place
            return W

        def worksheet_list_for_user(self, user, typ="active", sort='last_edited', reverse=False, search=None):
            X = self.get_worksheets_with_viewer(user)
            if typ == "trash":
                W = [x for x in X if x.is_trashed(user)]
            elif typ == "active":
                W = [x for x in X if x.is_active(user)]
            else: # typ must be archived
                W = [x for x in X if not (x.is_trashed(user) or x.is_active(user))]
            if search:
                W = [x for x in W if x.satisfies_search(search)]
            sort_worksheet_list(W, sort, reverse)  # changed W in place
            return W

        ##########################################################
        # Revision history for a worksheet
        ##########################################################
        def html_worksheet_revision_list(self, username, worksheet):
            r"""
            Return HTML for the revision list of a worksheet.

            INPUT:

            - ``username`` - a string

            - ``worksheet`` - an instance of Worksheet

            OUTPUT:

            - a string - the HTML for the revision list

            EXAMPLES::

                sage: from sagenb.flask_version import base # random output -- depends on warnings issued by other sage packages
                sage: app = base.create_app(tmp_dir(ext='.sagenb'))
                sage: ctx = app.app_context()
                sage: ctx.push()
                sage: nb = base.notebook
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: W.body()
                u'\n\n{{{id=1|\n\n///\n}}}'
                sage: W.save_snapshot('admin')
                sage: nb.html_worksheet_revision_list('admin', W)
                u'...Revision...Last Edited...ago...'
            """
            data = worksheet.snapshot_data()  # pairs ('how long ago', key)

            return template(os.path.join("html", "notebook", "worksheet_revision_list.html"),
                            data = data, worksheet = worksheet,
                            notebook = self,
                            username = username)

        def html_specific_revision(self, username, ws, rev):
            r"""
            Return the HTML for a specific revision of a worksheet.

            INPUT:

            - ``username`` - a string

            - ``ws`` - an instance of Worksheet

            - ``rev`` - a string containing the key of the revision

            OUTPUT:

            - a string - the revision rendered as HTML
            """
            t = time.time() - float(rev[:-4])
            time_ago = prettify_time_ago(t)

            filename = ws.get_snapshot_text_filename(rev)
            txt = bz2.decompress(open(filename).read())
            W = self.scratch_worksheet()
            W.set_name('Revision of ' + ws.name())
            W.delete_cells_directory()
            W.edit_save(txt)

            data = ws.snapshot_data()  # pairs ('how long ago', key)
            prev_rev = None
            next_rev = None
            for i in range(len(data)):
                if data[i][1] == rev:
                    if i > 0:
                        prev_rev = data[i - 1][1]
                    if i < len(data)-1:
                        next_rev = data[i + 1][1]
                    break

            return template(os.path.join("html", "notebook",
                                         "specific_revision.html"),
                            worksheet = W, # the revision, not the original!
                            username = username, rev = rev, prev_rev = prev_rev,
                            next_rev = next_rev, time_ago = time_ago)

        def html_share(self, worksheet, username):
            r"""
            Return the HTML for the "share" page of a worksheet.

            INPUT:

            - ``username`` - a string

            - ``worksheet`` - an instance of Worksheet

            OUTPUT:

            - string - the share page's HTML representation

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_share(W, 'admin')
                u'...currently shared...add or remove collaborators...'
            """
            return template(os.path.join("html", "notebook", "worksheet_share.html"),
                            worksheet = worksheet,
                            notebook = self,
                            username = username)

        def html_download_or_delete_datafile(self, ws, username, filename):
            r"""
            Return the HTML for the download or delete datafile page.

            INPUT:

            - ``username`` - a string

            - ``ws`` - an instance of Worksheet

            - ``filename`` - a string; the name of the file

            OUTPUT:

            - a string - the page rendered as HTML

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_download_or_delete_datafile(W, 'admin', 'bar')
                u'...Data file: bar...DATA is a special variable...uploaded...'
            """
            ext = os.path.splitext(filename)[1].lower()
            file_is_image, file_is_text = False, False
            text_file_content = ""

            if ext in ['.png', '.jpg', '.gif']:
                file_is_image = True
            if ext in ['.txt', '.tex', '.sage', '.spyx', '.py', '.f', '.f90', '.c']:
                file_is_text = True
                text_file_content = open(os.path.join(ws.data_directory(), filename)).read()
                # Detect the character encoding
                from BeautifulSoup import UnicodeDammit
                text_file_content = UnicodeDammit(text_file_content).unicode

            return template(os.path.join("html", "notebook", "download_or_delete_datafile.html"),
                            worksheet = ws, notebook = self,
                            username = username,
                            filename_ = filename,
                            file_is_image = file_is_image,
                            file_is_text = file_is_text,
                            text_file_content = text_file_content)

        ##########################################################
        # Accessing all worksheets with certain properties.
        ##########################################################
        def active_worksheets_for(self, username):
            # TODO: check if the worksheets are active
            #return [ws for ws in self.get_worksheets_with_viewer(username) if ws.is_active(username)]
            return self.users_worksheets_view(username)

        def get_all_worksheets(self):
            """
            We should only call this if the user is admin!
            """
            all_worksheets = []
            for username in self._user_manager.users():
                if username in ['_sage_', 'pub']:
                    continue
                for w in self.users_worksheets(username):
                    all_worksheets.append(w)
            return all_worksheets

        def get_worksheets_with_viewer(self, username):
            if self._user_manager.user_is_admin(username): return self.get_all_worksheets()
            return self.users_worksheets_view(username)

        def get_worksheets_with_owner(self, owner):
            return self.users_worksheets(owner)

        def get_worksheet_with_filename(self, filename):
            """
            Get the worksheet with the given filename.  If there is no
            such worksheet, raise a ``KeyError``.

            INPUT:

            - ``filename`` - a string

            OUTPUT:

            - a Worksheet instance
            """
            try:
                return self.__worksheets[filename]
            except KeyError:
                raise KeyError("No worksheet with filename '%s'" % filename)

        ###########################################################
        # Saving the whole notebook
        ###########################################################

        def save(self):
            """
            Save this notebook server to disk.
            """
            S = self.__storage
            S.save_users(self.user_manager().users())
            S.save_server_conf(self.conf())
            self._user_manager.save(S)
            # Save the non-doc-browser worksheets.
            for n, W in self.__worksheets.items():
                if not n.startswith('doc_browser'):
                    S.save_worksheet(W)
            if hasattr(self, '_user_history'):
                for username, H in iteritems(self._user_history):
                    S.save_user_history(username, H)

        def save_worksheet(self, W, conf_only=False):
            self.__storage.save_worksheet(W, conf_only=conf_only)

        def logout(self, username):
            r"""
            Do not do anything on logout (so far).

            In particular, do **NOT** stop all ``username``'s worksheets!
            """
            pass

        def delete_doc_browser_worksheets(self):
            for w in self.users_worksheets('_sage_'):
                if w.name().startswith('doc_browser'):
                    self.delete_worksheet(w.filename())

        ###########################################################
        # HTML -- generate most html related to the whole notebook page
        ###########################################################
        def html_plain_text_window(self, worksheet, username):
            r"""
            Return HTML for the window that displays a plain text version
            of the worksheet.

            INPUT:

            -  ``worksheet`` - a Worksheet instance

            -  ``username`` - a string

            OUTPUT:

            - a string - the plain text window rendered as HTML

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_plain_text_window(W, 'admin')
                u'...pre class="plaintext"...cell_intext...textfield...'
            """
            plain_text = worksheet.plain_text(prompts=True, banner=False)
            plain_text = escape(plain_text).strip()

            return template(os.path.join("html", "notebook", "plain_text_window.html"),
                            worksheet = worksheet,
                            notebook = self,
                            username = username, plain_text = plain_text,
                            MATHJAX = MATHJAX, JEDITABLE_TINYMCE = JEDITABLE_TINYMCE)

        def html_edit_window(self, worksheet, username):
            r"""
            Return HTML for a window for editing ``worksheet``.

            INPUT:

            - ``username`` - a string containing the username

            - ``worksheet`` - a Worksheet instance

            OUTPUT:

            - a string - the editing window's HTML representation

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_edit_window(W, 'admin')
                u'...textarea class="plaintextedit"...{{{id=1|...//...}}}...'
            """

            return template(os.path.join("html", "notebook", "edit_window.html"),
                            worksheet = worksheet,
                            notebook = self,
                            username = username)

        def html_beforepublish_window(self, worksheet, username):
            r"""
            Return HTML for the warning and decision page displayed prior
            to publishing the given worksheet.

            INPUT:

            - ``worksheet`` - an instance of Worksheet

            - ``username`` - a string

            OUTPUT:

            - a string - the pre-publication page rendered as HTML

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_beforepublish_window(W, 'admin')
                u'...want to publish this worksheet?...re-publish when changes...'
            """
            msg = """You can publish your worksheet to the Internet, where anyone will be able to access and view it online.
            Your worksheet will be assigned a unique address (URL) that you can send to your friends and colleagues.<br/><br/>
            Do you want to publish this worksheet?<br/><br/>
            <form method="get" action=".">
            <input type="hidden" name="yes" value="" />
            <input type="submit" value="Yes" style="margin-left:10px" />
            <input type="button" value="No" style="margin-left:5px" onClick="parent.location=\'../'"><br/><br/>
            <input type="checkbox" name="auto" style="margin-left:13px" /> Automatically re-publish when changes are made and saved
            </form>
            """
            return template(os.path.join("html", "notebook", "beforepublish_window.html"),
                            worksheet = worksheet,
                            notebook = self,
                            username = username)

        def html_afterpublish_window(self, worksheet, username, url, dtime):
            r"""
            Return HTML for a given worksheet's post-publication page.

            INPUT:

            - ``worksheet`` - an instance of Worksheet

            - ``username`` - a string

            - ``url`` - a string representing the URL of the published
              worksheet

            - ``dtime`` - an instance of time.struct_time representing the
              publishing time

            OUTPUT:

            - a string - the post-publication page rendered as HTML
            """
            from time import strftime
            time = strftime("%B %d, %Y %I:%M %p", dtime)

            return template(os.path.join("html", "notebook", "afterpublish_window.html"),
                            worksheet = worksheet,
                            notebook = self,
                            username = username, url = url, time = time)

        def html_upload_data_window(self, ws, username):
            r"""
            Return HTML for the "Upload Data" window.

            INPUT:

            - ``worksheet`` - an instance of Worksheet

            - ``username`` - a string

            OUTPUT:

            - a string - the HTML representation of the data upload window

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html_upload_data_window(W, 'admin')
                u'...Upload or Create Data File...Browse...url...name of a new...'
            """
            return template(os.path.join("html", "notebook", "upload_data_window.html"),
                            worksheet = ws, username = username)

        def html(self, worksheet_filename=None, username='guest', admin=False,
                 do_print=False):
            r"""
            Return the HTML for a worksheet's index page.

            INPUT:

            - ``worksheet_filename`` - a string (default: None)

            - ``username`` - a string (default: 'guest')

            - ``admin`` - a bool (default: False)

            OUTPUT:

            - a string - the worksheet rendered as HTML

            EXAMPLES::

                sage: nb = sagenb.notebook.notebook.Notebook(tmp_dir(ext='.sagenb'))
                sage: nb.create_default_users('password')
                sage: W = nb.create_new_worksheet('Test', 'admin')
                sage: nb.html(W.filename(), 'admin')
                u'...Test...cell_input...if (e.shiftKey)...state_number...'
            """
            if worksheet_filename is None or worksheet_filename == '':
                worksheet_filename = None
                W = None
            else:
                try:
                    W = self.get_worksheet_with_filename(worksheet_filename)
                except KeyError:
                    W = None

            from flask import current_app
            if W is None:
                return current_app.message(gettext("The worksheet does not exist"), username=username)

            if W.docbrowser() or W.is_published():
                if W.is_published() or self.user_manager().user_is_guest(username):
                    template_page = os.path.join('html', 'notebook', 'guest_worksheet_page.html')
                else:
                    template_page = os.path.join("html", "notebook", "doc_page.html")
            elif do_print:
                template_page = os.path.join('html', 'notebook', 'print_worksheet.html')
            else:
                template_page = os.path.join("html", "notebook", "worksheet_page.html")

            return template(template_page, worksheet = W,
                            notebook = self, do_print=do_print,
                            username = username)

        def upgrade_model(self):
            """
            Upgrade the model, if needed.

            - Version 0 (or non-existent model version, which defaults to 0): Original flask notebook
            - Version 1: shared worksheet data cached in the User object
            """
            model_version=self.conf()['model_version']
            if model_version is None or model_version<1:
                print("Upgrading model version to version 1")
                # this uses code from get_all_worksheets()
                user_manager = self.user_manager()
                num_users=0
                for username in self._user_manager.users():
                    num_users+=1
                    if num_users%1000==0:
                        print('Upgraded %d users' % num_users)
                    if username in ['_sage_', 'pub']:
                        continue
                    try:
                        for w in self.users_worksheets(username):
                            owner = w.owner()
                            id_number = w.id_number()
                            collaborators = w.collaborators()
                            for u in collaborators:
                                try:
                                    user_manager.user(u).viewable_worksheets().add((owner, id_number))
                                except KeyError:
                                    # user doesn't exist
                                    pass
                    except (UnicodeEncodeError, OSError):
                        # Catch UnicodeEncodeError because sometimes a username has a non-ascii character
                        # Catch OSError since sometimes when moving user directories (which happens
                        #   automatically when getting user's worksheets), OSError: [Errno 39] Directory not empty
                        #   is thrown (we should be using shutil.move instead, probably)
                        # users with these problems won't have their sharing cached, but they will probably have
                        # problems logging in anyway, so they probably won't notice not having shared worksheets
                        import sys
                        import traceback
                        print('Error on username %s' % username.encode('utf8'),
                              file=sys.stderr)
                        print(traceback.format_exc(), file=sys.stderr)
                        pass
                print('Done upgrading to model version 1')
                self.conf()['model_version'] = 1

    ####################################################################

    def load_notebook(dir, interface=None, port=None, secure=None, user_manager=None):
        """
        Load and return a notebook from a given directory.  Create a new
        one in that directory, if one isn't already there.

        INPUT:

        -  ``dir`` - a string that defines a directory name

        -  ``interface`` - the address of the interface the server listens at

        -  ``port`` - the port the server listens on

        -  ``secure`` - whether the notebook is secure

        OUTPUT:

        - a Notebook instance
        """
        if not dir.endswith('.sagenb'):
            if not os.path.exists(dir + '.sagenb') and os.path.exists(os.path.join(dir, 'nb.sobj')):
                try:
                    nb = migrate_old_notebook_v1(dir)
                except KeyboardInterrupt:
                    raise KeyboardInterrupt("Interrupted notebook migration.  Delete the directory '%s' and try again." % (os.path.abspath(dir+'.sagenb')))
                return nb
            dir += '.sagenb'

        dir = make_path_relative(dir)
        nb = Notebook(dir)
        nb.interface = interface
        nb.port = port
        nb.secure = secure

        # Install this copy of the notebook in misc.py as *the*
        # global notebook object used for computations.  This is
        # mainly to avoid circular references, etc.  This also means
        # only one notebook can actually be used at any point.
        import sagenb.notebook.misc
        sagenb.notebook.misc.notebook = nb

        return nb

    def migrate_old_notebook_v1(dir):
        """
        Back up and migrates an old saved version of notebook to the new one (`sagenb`)
        """
        nb_sobj = os.path.join(dir, 'nb.sobj')
        old_nb = pickle.loads(open(nb_sobj).read())

        ######################################################################
        # Tell user what is going on and make a backup
        ######################################################################

        print("")
        print("*" * 80)
        print("*")
        print("* The Sage notebook at")
        print("*")
        print("*      '%s'" % os.path.abspath(dir))
        print("*")
        print("* will be upgraded to a new format and stored in")
        print("*")
        print("*      '%s.sagenb'." % os.path.abspath(dir))
        print("*")
        print("* Your existing notebook will not be modified in any way.")
        print("*")
        print("*" * 80)
        print("")
        ans = raw_input("Would like to continue? [YES or no] ").lower()
        if ans not in ['', 'y', 'yes']:
            raise RuntimeError("User aborted upgrade.")

        # Create new notebook
        new_nb = Notebook(dir + '.sagenb')

        # Define a function for transfering the attributes of one object to another.
        def transfer_attributes(old, new, attributes):
            for attr_old, attr_new in attributes:
                if hasattr(old, attr_old):
                    setattr(new, attr_new, getattr(old, attr_old))

        # Transfer all the notebook attributes to our new notebook object

        new_nb.conf().confs = old_nb.conf().confs
        for t in ['pretty_print', 'server_pool', 'ulimit', 'system']:
            if hasattr(old_nb, '_Notebook__' + t):
                new_nb.conf().confs[t] = getattr(old_nb, '_Notebook__' + t)

        # Now update the user data from the old notebook to the new one:
        print("Migrating %s user accounts..." % len(old_nb.user_manager().users()))
        users = new_nb.user_manager().users()
        for username, old_user in iteritems(old_nb.user_manager().users()):
            new_user = user.User(old_user.username(), '',
                                 old_user.get_email(), old_user.account_type())
            new_user.set_hashed_password(old_user.password())
            transfer_attributes(old_user, new_user,
                                 [('_User__email_confirmed', '_email_confirmed'),
                                 ('_User__temporary_password', '_temporary_password'),
                                 ('_User__is_suspended', '_is_suspended')])
            # Fix the __conf field, which is also an instance of a class
            new_user.conf().confs = old_user.conf().confs
            users[new_user.username()] = new_user

        ######################################################################
        # Set the worksheets of the new notebook equal to the ones from
        # the old one.
        ######################################################################

        def migrate_old_worksheet(old_worksheet):
            """
            Migrates an old worksheet to the new format.
            """
            old_ws_dirname = old_ws._Worksheet__filename.partition(os.path.sep)[-1]
            new_ws = new_nb.worksheet(old_ws.owner(), old_ws_dirname)

            # some ugly creation of new attributes from what used to be stored
            tags = {}
            try:
                for user, val in iteritems(old_ws._Worksheet__user_view):
                    if isinstance(user, str):
                        # There was a bug in the old notebook where sometimes the
                        # user was the *module* "user", so we don't include that
                        # invalid data.
                        tags[user] = [val]
            except AttributeError:
                pass
            import time
            last_change = (old_ws.last_to_edit(), old_ws.last_edited())
            try:
                published_id_number = int(os.path.split(old_ws._Worksheet__published_version)[1])
            except AttributeError:
                published_id_number = None

            ws_pub = old_ws.worksheet_that_was_published().filename().split('/')
            ws_pub = (ws_pub[0], int(ws_pub[1]))

            obj = {'name': old_ws.name(), 'system': old_ws.system(),
                   'viewers': old_ws.viewers(),
                   'collaborators' :old_ws.collaborators(),
                   'pretty_print': old_ws.pretty_print(),
                   'ratings': old_ws.ratings(),
                   'auto_publish': old_ws.is_auto_publish(), 'tags': tags,
                   'last_change': last_change,
                   'published_id_number': published_id_number,
                   'worksheet_that_was_published': ws_pub
                   }

            new_ws.reconstruct_from_basic(obj)

            base = os.path.join(dir, 'worksheets', old_ws.filename())
            worksheet_file = os.path.join(base, 'worksheet.txt')
            if os.path.exists(worksheet_file):
                text = open(worksheet_file).read()
                # delete first two lines -- we don't embed the title or
                # system in the worksheet text anymore.
                i = text.find('\n')
                text=text[i+1:]
                i = text.find('\n')
                text=text[i+1:]
                new_ws.edit_save(text)

            # copy over the DATA directory and cells directories
            try:
                dest = new_ws.data_directory()
                if os.path.exists(dest):
                    shutil.rmtree(dest)
                shutil.copytree(old_ws.data_directory(), dest)
            except Exception as msg:
                print(msg)

            try:
                if os.path.exists(old_ws.cells_directory()):
                    dest = new_ws.cells_directory()
                    if os.path.exists(dest):
                        shutil.rmtree(dest)
                    shutil.copytree(old_ws.cells_directory(), dest)
            except Exception as msg:
                print(msg)

            return new_ws

        worksheets = WorksheetDict(new_nb)
        num_worksheets = len(old_nb._Notebook__worksheets)
        print("Migrating (at most) %s worksheets..." % num_worksheets)
        from sage.misc.all import walltime
        tm = walltime()
        i = 0
        for ws_name, old_ws in iteritems(old_nb._Notebook__worksheets):
            if old_ws.docbrowser(): continue
            i += 1
            if i % 25==0:
                percent = i / float(num_worksheets)
                # total_time * percent = time_so_far, so
                # remaining_time = total_time - time_so_far = time_so_far*(1/percent - 1)
                print("    Migrated %s (of %s) worksheets (about %.0f seconds remaining)" % (
                    i, num_worksheets, walltime(tm) * (1 / percent - 1)))
            new_ws = migrate_old_worksheet(old_ws)
            worksheets[new_ws.filename()] = new_ws
        new_nb._Notebook__worksheets = worksheets

        # Migrating history
        new_nb._user_history = {}
        for username in old_nb.user_manager().users().keys():
            history_file = os.path.join(dir, 'worksheets', username, 'history.sobj')
            if os.path.exists(history_file):
                new_nb._user_history[username] = pickle.loads(open(history_file).read())

        # Save our newly migrated notebook to disk
        new_nb.save()

        print("Worksheet migration completed.")
        return new_nb

    def make_path_relative(dir):
        r"""
        Replace an absolute path with a relative path, if possible.
        Otherwise, return the given path.

        INPUT:

        - ``dir`` - a string containing, e.g., a directory name

        OUTPUT:

        - a string
        """
        base, file = os.path.split(dir)
        if os.path.exists(file):
            return file
        return dir

    ##########################################################
    # Misc
    ##########################################################

    def sort_worksheet_list(v, sort, reverse):
        """
        Sort a given list on a given key, in a given order.

        INPUT:

        - ``sort`` - a string; 'last_edited', 'owner', 'rating', or 'name'

        - ``reverse`` - a bool; if True, reverse the order of the sort.

        OUTPUT:

        - the sorted list
        """
        f = None
        if sort == 'last_edited':
            def c(a):
                return a.last_edited()
            reverse = not reverse
            f = c
        elif sort == 'name':
            def c(a):
                return (a.name().lower(), -a.last_edited())
            f = c
        elif sort == 'owner':
            def c(a):
                return (a.owner().lower(), -a.last_edited())
            f = c
        elif sort == "rating":
            def c(a):
                return (a.rating(), -a.last_edited())
            reverse = not reverse
            f = c
        else:
            raise ValueError("invalid sort key '%s'" % sort)
        v.sort(key=f, reverse=reverse)