FetchHTML

/* Copyright 1997 Acorn Computers Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/***************************************************/
/* File   : FetchHTML.c                            */
/*                                                 */
/* Purpose: Fetch functions that deal with HTMLLib */
/*          data (getting more of it, freeing it,  */
/*          and so forth). Compare with higher     */
/*          higher level Fetch.c and FetchPage.c.  */
/*                                                 */
/* Author : A.D.Hodgkinson                         */
/*                                                 */
/* History: 17-Aug-97: Created from Fetch.c.       */
/***************************************************/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "swis.h"
#include "flex.h"

#include "HTMLLib.h" /* HTML library API, Which will include html2_ext.h, tags.h and struct.h */

#include "wimp.h"
#include "event.h"

#include "svcprint.h"
#include "Global.h"
#include "MiscDefs.h"
#include "Utils.h"

#include "About.h"
#include "Authorise.h"
#include "Browser.h"
#include "Cookies.h"
#include "Encoding.h"
#include "Fetch.h" /* (Which itself includes URLstat.h) */
#include "Filetypes.h"
#include "History.h"
#include "Save.h" /* (For Save_ScrapFile only)*/
#include "URLutils.h"
#include "URLveneer.h"

#include "FetchHTML.h"

/* Statics */

static char * fetch_buffer = NULL; /* Address of buffer for getting data from the URL module in html_get_next_token. */

/* Local definitions */

#define FetchBufferSize 8192 /* Size of buffer for getting data from the URL module in html_get_next_token. */

/* Local compilation options */

#undef DUMP_HEADERS

/*************************************************/
/* html_get()                                    */
/*                                               */
/* Fetches and optionally starts parsing HTML.   */
/*                                               */
/* Parameters: Pointer to URL to fetch;          */
/*                                               */
/*             Pointer to a pointer for extra    */
/*             data for POST etc. (allows this   */
/*             to be in a flex block);           */
/*                                               */
/*             Pointer to an int into which a    */
/*             handle for this fetch will be     */
/*             placed;                           */
/*                                               */
/*             The fetch method, e.g. POST or    */
/*             GET;                              */
/*                                               */
/*             Pointer to the user name for      */
/*             MailServ (if in a multiuser       */
/*             environment);                     */
/*                                               */
/*             1 to allow parsing, else 0;       */
/*                                               */
/*             1 to allow proxying, else 0 (e.g. */
/*             to force a refetch, rather than   */
/*             going via. a cache).              */
/*************************************************/

_kernel_oserror * html_get(char * url, char ** extradata, int * handle, int method,
                           char * user, int allowparse, int proxy)
{
  _kernel_oserror * e;
  int               ok;
  unsigned int      h;

  #ifdef TRACE
    if (tl & (1u<<6)) Printf("html_get: Called\n");
  #endif

  *handle = 0;

  /* Register the session with the URL module */

  e = url_register(0, &h);

// Sort out the proxying code properly!...

  /* Deal with proxying if necessary */

  if (!e && choices.use_proxy)
  {
    char   method[64];
    char * method_ptr;
    int    method_len;

    /* Extract the fetch method from the proxy address */

    method_ptr = strstr(choices.proxy_address, ":");

    if (method_ptr)
    {
      method_len = (int) method_ptr - (int) choices.proxy_address + 1;
      if (method_len > sizeof(method) - 1) method_len = sizeof(method) - 1;
      strncpy(method, choices.proxy_address, method_len);
      method[method_len] = 0;
    }
    else strncpy(method, "http:", sizeof(method));

    e = url_set_proxy(0, h, choices.proxy_address, method, 0);
  }

  if (!e)
  {
    urlstat * up;

    #ifdef TRACE
      if (tl & (1u<<6)) Printf("html_get: Session registered, ID is %d\n",h);
    #endif

    e = urlstat_add_entry(1, &up);

    if (e)
    {
      url_deregister(0,h);
      return e;
    }

    /* Initialise the new structure */

    up->session    = *handle = (int) h; /* The fetch's session handle                      */
    up->type       = TYPE_HTMLFILE;     /* Type of file - state it is an HTML file for now */
    up->fetching   = 1;                 /* We are still fetching                           */
    up->method     = method;            /* Current fetch method                            */
    up->extradata  = NULL;              /* Filled in later, if there is extra data         */
    up->allowparse = allowparse;        /* Do we parse the data?                           */

    /* If there is any extra data for POST or whatever, deal with it. */
    /* The POST request entries must come first in the extra header   */
    /* info, so that the browser can make the assumption that         */
    /* everything from Content-Type forwards may be stripped in the   */
    /* event of a redirection when the current fetch method is POST.  */

    if (extradata && *extradata)
    {
      int len;

      len = strlen(*extradata);

      /* Allocate space for the extra data, the anchor stored in up->extradata */

      #ifdef TRACE
        if (tl & (1u<<12)) Printf("html_get: flex_alloc %d for 'extradata' store\n",len + 3);
      #endif

      if (!flex_alloc((flex_ptr) &up->extradata, len + 3))
      {
        url_deregister(0,h);

        return make_no_fetch_memory_error(2);
      }
      else
      {
        char head[80];

        #ifdef TRACE
          flexcount += (len + 3);
          if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
        #endif

        /* CR+LF into the top of the new block of memory */

        up->extradata[0] = '\r';
        up->extradata[1] = '\n';

        /* Copy the extra data under the CR+LF */

        strcpy(up->extradata + 2, *extradata);

        /* Header entry for the extra data - again, the removal routines in the */
        /* fetcher's redirection code assume that this comes after Content-Type */
        /* and the body content comes after this, to make life easy there.      */

        sprintf(head, "Content-Length: %d", len);

        /* Insert the header entries above the extra data already in the block. */

        ok = html_insert_header(head, (flex_ptr) &up->extradata);

        /* (html_insert_header returns 1 for success, 0 for memory claim failure) */

        if (!ok)
        {
          url_deregister(0, h);

          return make_no_fetch_memory_error(3);
        }

        StrNCpy0(head, "Content-Type: application/x-www-form-urlencoded");
        ok = html_insert_header(head, (flex_ptr) &up->extradata);

        if (!ok)
        {
          url_deregister(0, h);

          return make_no_fetch_memory_error(4);
        }
      }
    }

    #ifndef SINGLE_USER

      /* If user details are given, insert the appropriate header entry */

      if (user)
      {
        char head[Limits_Multi_UserName + 80];

        sprintf(head, "Mailserv-User: %s", user);
        ok = html_insert_header(head, (flex_ptr) &up->extradata);

        if (!ok)
        {
          url_deregister(0, h);

          return make_no_fetch_memory_error(5);
        }
      }

    #endif

    /* If we aren't to use a proxy - actually, this is Customer-speak */
    /* for no cache - say so in the header, in various ways.           */

    if (!proxy)
    {
      /* For Customer compatability first, most servers will get the */
      /* second, and newer servers prefer the last.                   */

      ok         = html_insert_header("X-NoProxy:",             (flex_ptr) &up->extradata);
      if (ok) ok = html_insert_header("Pragma:no-cache",        (flex_ptr) &up->extradata);
      if (ok) ok = html_insert_header("Cache-Control:no-cache", (flex_ptr) &up->extradata);

      if (!ok)
      {
        url_deregister(0, h);

        return make_no_fetch_memory_error(6);
      }
    }

    /* Last but not least, do, er, something... */

    {
      char   c = 0;
      char * p = NULL;

      /* If non-zero on exit, p will point to the position of a hash */
      /* in the URL (i.e., this finds out if an anchor is specified) */

      p = fetch_find_name_tag(url);

      /* If there is a hash, turn it into a zero for now so the string */
      /* contains just the URL and not the anchor.                     */

      if (p) c = *p, *p = 0;

      e = url_get_url(URL_GetURL_AgentGiven, /* Should use a custom User Agent */
                      h,                     /* Session handle                 */
                      method,                /* Fetch method                   */
                      url,                   /* URL to get                     */
                      &up->extradata,        /* Any extra data for POST etc.   */
                      NULL,                  /* (Would be a status word)       */
                      2);                    /* Mode; 2 = header and data      */

      /* Put the hash back if was removed earlier. */

      if (p) *p = c;
    }
  }

  #ifdef TRACE
    if (tl & (1u<<6))
    {
      if (!e) Printf("html_get: Successful\n");
      else Printf("html_get: Exitting with an error\n");
    }
  #endif

  return e;
}

/*************************************************/
/* html_insert_header()                          */
/*                                               */
/* Inserts a string into the header for an HTML  */
/* fetch (for POST). Puts it at the top.         */
/*                                               */
/* Parameters: Pointer to the null terminated    */
/*             string to insert (this ends up    */
/*             CR+LF terminated in the header);  */
/*                                               */
/*             Pointer to a flex anchor, which   */
/*             points to existing header data or */
/*             is NULL if there is no header at  */
/*             the time of the function call.    */
/*                                               */
/* Returns:    1 if successful, or 0; you must   */
/*             externally generate an error      */
/*             appropriate to the memory claim   */
/*             having failed.                    */
/*************************************************/

int html_insert_header(char * header, flex_ptr data)
{
  if (header)
  {
    int ok, s, len;

    len = strlen(header) + 2;

    /* 'data' points to an anchor; if this isn't null, find the */
    /* size of the block the anchor points to                   */

    if (*data) s = flex_size(data);
    else s = 0;

    /* If the block is > 0 bytes, extend it to a block big      */
    /* enough to hold the extra header data, else allocate a    */
    /* new block to hold it. Note that s will be zero if a new  */
    /* block was allocated, else it holds the old block size.   */

    #ifdef TRACE
      if (tl & (1u<<12))
      {
        if (s) Printf("html_insert_header: flex_extend to %d for header store\n",len + s);
        else   Printf("html_insert_header: flex_alloc %d for header store\n",len + 1);
      }
    #endif

    if (s) ok = flex_extend(data, len + s);
    else   ok = flex_alloc(data, len + 1); /* Note len *plus 1*. */

    if (!ok) return 0;

    #ifdef TRACE
      if (s) flexcount += len;
      else   flexcount += (len + 1);
      if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
    #endif

    /* Shuffle the header data down to make room for the new    */
    /* stuff at the top, if there was any data there to move.   */

    #ifdef TRACE
      if (tl & (1u<<18)) Printf("\0213html_insert_header: memove from %p to %p for %d bytes\0217\n",((int) (*data)) + len, *data, s);
    #endif

    if (s) memmove((void *) (((int) (*data)) + len), *data, s);

    /* Copy the new data into the top of the header. Don't want */
    /* to overflow so use strncpy for extra caution...          */

    strncpy(*data, header, len - 2);

    /* Terminate the string with CR+LF                          */

    ((char *) (*data))[len - 2] = '\r';
    ((char *) (*data))[len - 1] = '\n';

    /* If s is zero, i.e. a new block was created here, make    */
    /* sure it ends in zero (so C will think the string has     */
    /* ended properly if a string is read from the buffer). We  */
    /* can reference (array)[len] as the block allocation was   */
    /* done to len plus 1 bytes (see above).                    */

    if (!s) ((char *) (*data))[len] = 0;
  }

  return 1;
}

/*************************************************/
/* html_close()                                  */
/*                                               */
/* Closes the specified handle, aborting any     */
/* fetch and freeing up memory relating to it.   */
/*                                               */
/* Parameters: A fetch handle (usually from the  */
/*             browser_data->fetch_handle        */
/*             field).                           */
/*************************************************/

_kernel_oserror * html_close(int handle)
{
  urlstat * up;

  #ifdef TRACE
    if (tl & (1u<<6)) Printf("html_close: Called\n");
  #endif

  url_deregister(0, handle);

  /* It's another linked list traversal... As long as we aren't at the */
  /* end of the list, and we haven't reached the item relating to this */
  /* fetch, keep looking.                                              */

  up = urlstat_find_entry(handle);

  /* After the above loop, 'up' points to the structure for this */
  /* fetch or is null; in the latter case, give an error.        */

  if (!up)
  {
    erb.errnum = Utils_Error_Custom_Fatal;

    StrNCpy0(erb.errmess,
             lookup_token("StrNotFd:Internal error: Can't find structure in %0.",
                          0,
                          "html_close"));

    #ifdef TRACE
      if (tl & (1u<<6)) Printf("html_close: Exiting with error\n");
    #endif

    return &erb;
  }

  /* If there is HTMLLib derived data attached, deal with this */

  if (up->stream)
  {
    unsigned int   context = HtmlReturnContext(up->stream);
    browser_data * browser = last_browser;

    /* Should Never Happen...! */

    if (!context)
    {
      erb.errnum = Utils_Error_Custom_Fatal;

      StrNCpy0(erb.errmess,
               lookup_token("NoContxt:Serious internal error - Block is already free or was not HtmlAlloc'd in html_close; must exit immediately.",
                            0,
                            0));

      return &erb;
    }

    /* Ensure that any HStream pointers inside any current browser_data */
    /* structures are not part of this stream - if so, clear them.      */

    while (browser)
    {
      /* For now, just a few selected items */

      if (browser->selected && HtmlReturnContext(browser->selected) == context)
      {
        browser_clear_selection(browser, 0);
        browser->selected = NULL;
      }

      if (browser->highlight && HtmlReturnContext(browser->highlight) == context)
      {
        browser_clear_highlight(browser, 0);
        browser->highlight = NULL;
      }

      if (browser->pointer_over && HtmlReturnContext(browser->pointer_over) == context) browser->pointer_over = NULL;

      browser = browser->previous;
    }

    #ifdef TRACE
      if (up->stream)
      {
        if (tl & (1u<<12)) Printf("html_close: Calling HtmlStreamFree on %p\n",up->stream);
        if (tl & (1u<<18)) Printf("\0212Closing stream %p\0217\n",up->stream);
      }
    #endif

    HtmlStreamFree(up->stream);
  }

  /* If there is extra context data allocated, free it */

  if (up->context)
  {
    #ifdef TRACE
      if (tl & (1u<<12)) Printf("html_close: free block %p for 'context' field of 'urlstat' structure\n",up->context);
    #endif

    HtmlEndParse(up->context);
    up->context = NULL;
  }

  #ifdef TRACE
    if (up->extradata)
    {
      if (tl & (1u<<12)) Printf("html_close: flex_free block %p for 'extradata' field of 'urlstat' structure\n",&up->extradata);
      flexcount -= flex_size((flex_ptr) &up->extradata);
      if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
    }
  #endif

  /* If there is extra flex data attched, free this too */

  if (up->extradata) flex_free((flex_ptr) &up->extradata);

  /* Finally, get rid of the structure itself */

  urlstat_remove_entry(up);

  #ifdef TRACE
    if (tl & (1u<<6)) Printf("html_close: Successful\n");
  #endif

  return NULL;
}

/*************************************************/
/* html_get_next_token()                         */
/*                                               */
/* Gets a chunk of document source from a given  */
/* fetch handle, and may generate new HStream    */
/* structures as the document is passed over to  */
/* the HTML library parser.                      */
/*                                               */
/* Parameters: Pointer to a browser_data struct  */
/*             relevant to the fetch or NULL;    */
/*                                               */
/*             The fetch handle;                 */
/*                                               */
/*             Pointer to int into which the     */
/*             number of bytes still to be       */
/*             fetched is played;                */
/*                                               */
/*             Pointer to int into which the     */
/*             number of bytes fetched so far is */
/*             placed;                           */
/*                                               */
/*             Pointer to an HStream *, into     */
/*             which the address of the base of  */
/*             the token list is written, or     */
/*             NULL to signal 'not ready';       */
/*                                               */
/*             Pointer to an int, into which a   */
/*             reason code is placed:            */
/*                                               */
/*             0: Token has been received OK,    */
/*             1: We are waiting for something,  */
/*             2: A redirect has been detected   */
/*                (in this case, *remaining will */
/*                point at the new URL),         */
/*             3: This data is not parseable (in */
/*                this case, *remaining holds a  */
/*                filetype);                     */
/*                                               */
/*             Pointer to pointer to the store   */
/*             for the whole of the data fetched */
/*             so far (if any), be it an HTML    */
/*             document, image, or whatever;     */
/*                                               */
/*             Pointer to string holding the URL */
/*             that is being fetched;            */
/*                                               */
/*             1 if this is an image fetch, else */
/*             0 for HTML or unknown.            */
/*                                               */
/* Assumes:    That if the browser_data struct   */
/*             pointer is NULL, the fetch is not */
/*             for an internal URL. The other    */
/*             pointers must NOT be NULL unless  */
/*             it is specifically stated that    */
/*             they may be in the parameters     */
/*             list.                             */
/*************************************************/

_kernel_oserror * html_get_next_token(browser_data * b, unsigned int handle, int * remaining, int * size,
                                      HStream ** token, int * waiting, flex_ptr source, char * url, int image)
{
  _kernel_oserror * e = NULL;
  int               r = 0;
  urlstat         * up;
  char              ref_url[Limits_URL];

  #ifdef TRACE
    if (tl & (1u<<6)) Printf("html_get_next_token: Called\n");
  #endif

  /* Start in the default state of having no HStream to pass back */
  /* through *token.                                              */

  if (token) *token = NULL;

  /* Until we know better, signal that we're waiting */

  *waiting = 1;

  /* Ensure a fetch buffer is allocated */

  if (!fetch_buffer)
  {
    fetch_buffer = malloc(FetchBufferSize); /* See top of this file */

    if (!fetch_buffer)
    {
      #ifdef TRACE
        if (tl & (1u<<6)) Printf("html_get_next_token: Exiting with error\n");
      #endif

      return make_no_cont_memory_error(8);
    }
  }

  /* Get the urlstat structure for this fetch */

  up = urlstat_find_entry(handle);

  if (!up)
  {
    erb.errnum = Utils_Error_Custom_Fatal;

    StrNCpy0(erb.errmess,
             lookup_token("StrNotFd:Internal error: Can't find structure in %0.",
                          0,
                          "html_get_next_token"));

    #ifdef TRACE
      if (tl & (1u<<6)) Printf("html_get_next_token: Exiting with error\n");
    #endif

    return &erb;
  }

  /* Only look for an anchor and use url_read_data for URLs which */
  /* are not internal or are direct data saves.                   */

  StrNCpy0(ref_url, url);

  if (image || b->displayed == Display_Fetched_Page || b->save_link)
  {
    int status = 0;

    /* Want to make sure we work on a URL which doesn't */
    /* have an anchor in it, so copy over the url to    */
    /* a local buffer and if there's a '#' marking an   */
    /* anchor, replace it with a string terminator.     */

    char * p = fetch_find_name_tag(ref_url);

    if (p) * p = 0;

    /* If there isn't an authorisation request in progress, and the  */
    /* fetch is apparently in progress, and the authorisation status */
    /* isn't '1' (which means 'doing'), get some data from the URL   */
    /* module. The url_read_data call puts the number of bytes read  */
    /* into r.                                                       */

    if (!authorising && up->fetching && (up->authorised != 1))
    {
      e = url_read_data(0,               /* Flags - must be 0 at present */
                        handle,          /* Session handle               */
                        fetch_buffer,    /* Buffer to receive data       */
                        FetchBufferSize, /* The buffer's size            */

                        &status,         /* Protocol status              */
                        &r,              /* Number of bytes read         */

                        remaining);      /* Number of bytes left to get  */

      /* Deal with cookies */

      if (!e && (status & (1u<<16))) e = cookies_process_cookie(b);

      /* Deal with the size information */

      if (*remaining > 0 && !b->data_size) b->data_size = *remaining + r;
    }
  }
  else if (up->fetching) /* Don't repeat this over and over... */
  {
    /* This is an internal URL, so treat specially */

    int    ok;
    char * extra = "";
    char * tail  = "";
    int    len, exoff, toff;

    if (*source) flex_free(source);

    /* Work out the length that the HTML file we're about to generate will be */

    switch (b->displayed)
    {
      case Display_External_Image:
      {
        int protolen;

        /* Look up the token embedded in the URL */

        lookup_token(url + Int_URL_Len, 1, 0);

        /* Find a ':' separating extra information and point just past it */

        exoff = urlutils_internal_extra(url);
        if (exoff) extra = url + exoff;

        len = strlen(tokens) + 1;

        if (*extra)
        {
          toff = urlutils_internal_tail(url);
          if (toff) tail = url + toff;
        }

        /* Is this a system variable name for fetching, e.g. Wimp$Scrap? */

        protolen = strlen(FileMethod ProtocolSepShort "<"); /* (URLutils.h) */

        if (
             !strncmp(extra,
                      FileMethod ProtocolSepShort "<",
                      protolen)
             &&
             extra[strlen(extra) - 1] == '>'
           )
        {
          int              required;
          _kernel_swi_regs r;

          /* We'll have to trash the ref_url block. We know ref_url is at     */
          /* least as long as URL, so extract the system variable name to it. */

          strncpy(ref_url, extra + protolen, strlen(extra) - protolen - 1); /* -1 to skip the closing '>' */
          ref_url[strlen(extra) - protolen - 1] = 0;

          /* Now find out how long the expanded form would be */

          r.r[0] = (int) ref_url;
          r.r[1] = (int) NULL;
          r.r[2] = -1;
          r.r[3] = 0;
          r.r[4] = 0;

          /* _swix will not work correctly for this particular SWI if */
          /* requiring the returned R2 value. Something to do with    */
          /* the call relying on generating an error, but _swix spots */
          /* it and pulls out earlier than the call expects. Or some  */
          /* such thing...                                            */

          _kernel_swi(OS_ReadVarVal, &r, &r);

          required = -r.r[2];

          /* Woah - system variable wasn't defined... */

          if (!required)
          {
            erb.errnum = Utils_Error_Custom_Normal;

            strcpy(erb.errmess, "<");
            strcat(erb.errmess, ref_url);
            strcat(erb.errmess, ">");
            strcat(erb.errmess, lookup_token("NotDefined: not defined.",0,0));

            return &erb;
          }

          if (required < 0 || required >= sizeof(ref_url) - 1) /* -1 = allow for terminator */
          {
            /* Well, we sort of haven't got enough memory. Ahem. */

            return make_no_memory_error(8);
          }

          /* Otherwise, expand the variable (_swix is OK here as we don't */
          /* want any returned register value).                           */

          _swix(OS_ReadVarVal,
                _INR(0,4),

                ref_url,
                ref_url,
                sizeof(ref_url),
                0,
                4);

          /* Ensure it is terminated correctly */

          ref_url[required - 1] = 0;

          /* Turn it into a fetchable URL */

          urlutils_pathname_to_url(ref_url, sizeof(ref_url));

          /* Point to this URL */

          extra = url = ref_url;
        }

        /* Note that this is a very slow function call... */

        len = utils_len_printf(tokens, extra, extra, tail);

        if (len < 0)
        {
          /* If the above fails, do our best to calculate the length. */
          /* This will always overestimate the size (safer to do this */
          /* than underestimate!).                                    */

          len = strlen(tokens) + 1;

          /* For external images, need to fit the extra data in twice, and */
          /* try to find a filename separator for a picture caption (put   */
          /* this in 'tail').                                              */

          if (*extra) len += strlen(extra) * 2 + strlen(tail) + 2;
        }
      }
      break;

      case Display_Scrap_File:
      {
        int found, type;

        if (*source) flex_free(source);

        /* Find the file length */

        _swix(OS_File,
              _INR(0,1) | _OUT(0) | _OUT(4) | _OUT(6),

              23, /* Read catalogue info for named, stamped object */
              Save_ScrapFile,

              &found,
              &len,
              &type);

        if (found != 1)
        {
          erb.errnum = Utils_Error_Custom_Normal;

          /* Error message will either be 'can't find the page', or, if this is */
          /* a frame and the frame source matches the fetching URL, 'can't find */
          /* the frame'.                                                        */

          if (
               b->ancestor          &&
               b->frame             &&
               b->frame->src        &&
               browser_fetch_url(b) &&
               !strcmp(b->frame->src, browser_fetch_url(b))
             )
          {
            StrNCpy0(erb.errmess,
                     lookup_token("WhatFrame:The sending application could not supply the page contents for this frame.",
                                  0,
                                  0));
          }
          else
          {
            StrNCpy0(erb.errmess,
                     lookup_token("WhatScrap:Cannot find the page to load; the sending application may have died.",
                                  0,
                                  0));
          }

          return &erb;
        }

        /* Is this a text file? */

        if (type == FileType_TEXT)
        {
          b->page_is_text = 1;
          up->type        = TYPE_TEXTFILE;
        }
        else b->page_is_text = 0;
      }
      break;

      case Display_About_Page:
      {
        len = 0;
      }
      break;
    }

    /* If required, claim memory for the page; complain if this fails */

    if (len) ok = flex_alloc(source, len);
    else     ok = 1;

    if (!ok)
    {
      #ifdef TRACE
        if (tl & (1u<<6)) Printf("html_get_next_token: Exiting with error\n");
      #endif

      return make_no_cont_memory_error(1);
    }

    /* Construct the page in the claimed block (or build it in a */
    /* new block).                                               */

    switch (b->displayed)
    {
      case Display_About_Page:
      {
        RetError(about_build_page(source));
      }
      break;

      case Display_External_Image:
      {
        memset(*source, 0, len);
        sprintf(*source, tokens, extra, extra, tail);
      }
      break;

      case Display_Scrap_File:
      {
        FILE * file;

        /* Load and delete the scrap file */

        file = fopen(Save_ScrapFile, "rb");

        if (!file) RetLastE;

        if (fread(*source, 1, len, file) < len)
        {
          fclose(file);
          RetLastE;
        }

        fclose(file);
        remove(Save_ScrapFile);
      }
      break;
    }

    /* Set up fetch flags to say that a fetch has been completed; since */
    /* we've filled in the document source store here, say that zero    */
    /* bytes have been fetched (otherwise code below will try to copy   */
    /* data out of the fetch_buffer block).                             */

    r              = 0;
    remaining      = 0;
    up->identified = 1;
    up->allowparse = 1;
    up->fetched    = 1;
    up->fetching   = 0;
  }

  /* If there isn't an error, and more than zero bytes have been read, */
  /* deal with the data (if any) returned from the above call.         */

  if (r && !e)
  {
    int ok, oldsize;

    /* 'fetched' is a flag which if set indicates at least 1 byte has been */
    /* got so far. If fetched is zero, and there is data in the source     */
    /* store (i.e. 'source' is not NULL) then free up the store as it does */
    /* not hold any valid data (must be from an old fetch).                */

    if (!up->fetched && *source)
    {
      #ifdef TRACE
        if (tl & (1u<<12)) Printf("html_get_next_token: (1) flex_free block %p which held page source\n",source);
        flexcount -= flex_size(source);
        if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
      #endif

      flex_free(source);
      *source = NULL;
    }

    /* Signal that there's definitely data fetched now. */

    up->fetched = 1;

    /* If there's store allocated at this point, it holds valid source; extend */
    /* it by the number of bytes read from the url_read_data call. Else, alloc */
    /* a new buffer to hold the data.                                          */

    #ifdef TRACE
      if (tl & (1u<<12))
      {
        if (*source) Printf("html_get_next_token: flex_extend by %d to %d for page source store\n",r,flex_size(source) + r);
        else         Printf("html_get_next_token: flex_alloc %d for page source store\n",r);
      }
    #endif

    if (*source)
    {
      oldsize = flex_size(source);
      ok = flex_extend(source, oldsize + r);
    }
    else
    {
      oldsize = 0;
      ok = flex_alloc(source, r);
    }

    #ifdef TRACE
      flexcount += r;
      if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
    #endif

    if (size) *size = oldsize + r;

    /* Report an error if the allocation failed */

    if (!ok)
    {
      #ifdef TRACE
        if (tl & (1u<<6)) Printf("html_get_next_token: Exiting with error\n");
      #endif

      return make_no_cont_memory_error(1);
    }

    /* The data block has been created/extended successfully, so copy the */
    /* data from the url_read_data call into it.                          */

    #ifdef TRACE
      if (tl & (1u<<18)) Printf("\0216html_get_next_token: memcpy from %p to %p for %d bytes\0217\n",((char *) (*source)) + oldsize, fetch_buffer, r);
    #endif

    memcpy(((char *) (*source)) + oldsize, fetch_buffer, r);
  }

  /* If we're not authorising the transfer and data has been fetched, proceed normally */

  if (!authorising && up->fetched)
  {
    unsigned int   hf  = 0;
    HStream      * new = NULL;

    /* If the stream has been identified as HTML... */

    if (up->identified)
    {
      /* Really need to make our minds up at this point about the */
      /* encoding. fetch_start may have set priority to default   */
      /* to clear out any old HTTP header info, but without       */
      /* changing the encoding, as the menu should only change    */
      /* when the document comes in. That's now, folks.           */

      if (b->encoding_priority == priority_default)
      {
        b->encoding = choices.encoding;
        encoding_update_menus(b);
      }

      /* If there's no parsing context, get one by calling HtmlParse - */
      /* this initialises the HTML parser, getting it ready to parse a */
      /* document (though it need not be present at this stage).       */
      /*                                                               */
      /* First time round, this won't be called as the stream hasn't   */
      /* been identified with HtmlIdentify yet.                        */

      if (up->context == NULL)
      {
        up->context = HtmlParse(ref_url,                /* Full page URL, so parser can handle relative links */
                                0,                      /* Length of document - zero at present (not known)   */
                                up->type,               /* Return type from the HtmlIdentify call             */
                                choices.support_frames, /* 1 to have FRAMESETs parsed, else 0                 */
                                choices.support_object, /* 1 to handle OBJECT etc., else get alt. HTML stream */
                                choices.support_tables, /* 1 to handle TABLE etc., else 0                     */
                                b->encoding,
                                b->encoding_priority);

        if (up->context) HtmlSetEncodingCallback(up->context, encoding_changed_by_meta, b);

        r = *source ? flex_size(source) : 0;
      }

      /* If there is new data in the source store (size = r) and no error at */
      /* present, attempt to parse the chunk of data with HtmlGetStream.     */

      if (r && !e)
      {
        new = HtmlGetStream(up->context,        /* Parser context, from HtmlParse            */
                            (char **) source,   /* Pointer to start of the complete document */
                            r,                  /* Size of the chunk that has been added     */
                            &hf);               /* Flags from HTMLLib, e.g. 'have more data' */

        up->stream = new;

        #ifdef TRACE
          if (tl & (1u<<18)) Printf("\0211(New stream for %p, %p)\0217\n", b, up->stream);
        #endif
      }

      if (!new)
      {
        /* There are no new HTML library structures */

        if (up->lasttoken)
        {
          /* There is no new data, but lasttoken indicates there are more tokens */
          /* left in the token stream from earlier calls that haven't been dealt */
          /* with. So move to the next one.                                      */

          up->lasttoken     = up->lasttoken->next;
          if (token) *token = up->lasttoken;
        }
      }
      else
      {
        /* There are some new HTML library structures. */

        if (!(hf & HTML_GOT_MORE_IN_A_TABLE))
        {
          /* The flag is unset, so the structures were added to the main token */
          /* stream and not to part of a table structure.                      */

          if (up->lasttoken)
          {
            /* Even though there are new structures, we still have older ones */
            /* that are not dealt with, so move to the next one (the remote   */
            /* server is sending us data than we're processing it - yay!).    */

            up->lasttoken     = up->lasttoken->next;
            if (token) *token = up->lasttoken;
          }
          else
          {
            /* There are no earlier structures left to deal with, so start on */
            /* the first of the new batch.                                    */

            up->lasttoken     = new;
            if (token) *token = up->lasttoken;
          }
        }
        else
        {
          /* The HTML_GOT_MORE_IN_A_TABLE flag is set, so structures were added to */
          /* a table arrangement, as well as (possibly) the main stream after it.  */

          if (!up->lasttoken)
          {
            /* We weren't waiting to process anything from an earlier call, so */
            /* start on this new table structure.                              */

            up->lasttoken     = new;
            if (token) *token = up->lasttoken;
          }
          else
          {
            /* We have undealt with structures from a previous fetch. Now, if */
            /* we are already on the same table structure as returned by the  */
            /* HtmlGetStream call, then stay there (i.e. process the new data */
            /* inside the table). Otherwise, move on.                         */

            if (up->lasttoken != new) up->lasttoken = up->lasttoken->next;
            if (token) *token = up->lasttoken;
          }
        }
      }

      /* If we've moved on to, or were already on no token, then whether or */
      /* not the fetch is still in progress determines whether or not we're */
      /* waiting. Otherwise, we aren't waiting for anything.                */

      if (!up->lasttoken) *waiting = !!up->fetching;
      else                *waiting = 0;
    }
    else if (up->authorised != 1)
    {
      /* The stream hasn't been identified as HTML, text or whatever, */
      /* but there isn't an authorisation in progress.                */

      int    s, o = 0;
      char * redirect;
      int    code;
      int    type;
      int    parseable;

      /* Get the fetch status */

      if (image || b->displayed == Display_Fetched_Page || b->save_link)
      {
        e = url_status(0, handle, &s, NULL, NULL);
        if (e) return e;
      }
      else s = URL_Status_Done;

      redirect  = NULL;
      type      = TYPE_UNKNOWN;
      parseable = 0;

      /* HttpStripHeaders, when passed a pointer to some document data, */
      /* and an offest into that stream, returns the offset into the    */
      /* stream at which it starts assuming HTTP style headers (if      */
      /* there is such a point).                                        */

      o = *source ? HttpStripHeaders((char *) *source, flex_size(source)) : 0;

      /* If o is 0, there were no HTTP headers. If o is -1, there wasn't */
      /* enough data to tell. Else, there were headers, and o is the     */
      /* offset into the stream of the data that follows those headers.  */

      if (o > 0)
      {
        int encoding;

        #ifdef DUMP_HEADERS
          {
            FILE * file;
            int    byte;

            file = fopen("<Wimp$ScrapDir>.Headers", "ab");

            if (file)
            {
              if (!image) fprintf(file, "For URL '%s', received header:\r\n\r\n", url);
              else        fprintf(file, "For image '%s', received header:\r\n\r\n", url);

              for (byte = 0; byte < o; byte++)
              {
                fputc((int) (*((char *) (((int) *source) + byte))), file);
              }

              fclose(file);
            }
          }
        #endif

        /* There are HTTP style headers in the data */
        /* stream; try to identify that stream.     */

        code = HtmlIdentify(ref_url,                     /* Allow relative redirections to work  */
                            (char *) *source,            /* Pointer to head of data stream       */
                            flex_size(source),           /* Amount of data in the stream         */
                            (s & URL_Status_Done) != 0,  /* Is it a complete stream? 1 = yes     */
                            &redirect,                   /* Will point to a URL if code = 302    */
                            &type,                       /* Will hold a filetype                 */
                            &parseable,                  /* Will say if the data is parseable    */
                            &encoding);                  /* Will say if it specified an encoding */

        /* Set the encoding if specified in the HTTP header */

        if (encoding && up->allowparse && b->encoding_priority < priority_http)
        {
          b->encoding_priority = priority_http;
          b->encoding          = encoding;

          encoding_update_menus(b);
        }

        /* Discard the stuff before the HTTP style headers by moving the data over them */

        if (o != flex_size(source))
        {
          #ifdef TRACE
            if (tl & (1u<<18)) Printf("\0213html_get_next_token: memove from %p to %p for %d bytes\0217\n",
                                    *source,
                                    (char*) (((int) *source) + o),
                                    flex_size(source) - o);
          #endif

          memmove(*source,
                  (char*) (((int) *source) + o),
                  flex_size(source) - o);

          /* Set o to the size of the data stream that is now in use, */
          /* and shrink the source store to this size.                */

          #ifdef TRACE
            if (tl & (1u<<12)) Printf("html_get_next_token: flex_extend to shrink source code store by %d to %d\n",o,flex_size(source) - o);
            flexcount -= o;
            if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
          #endif
        }

        o = flex_size(source) - o;

        /* If the size of the store minus o is less than 0, HttpStripHeaders */
        /* has failed completely and we must get out before everything else  */
        /* comes down...!                                                    */

        if (o < 0)
        {
          erb.errnum = Utils_Error_Custom_Fatal;

          StrNCpy0(erb.errmess,
                   lookup_token("HSHOvrrn:Serious internal error - HttpStripHeaders has failed; must exit immediately.",
                                0,
                                0));

          show_error_cont(&erb); /* This will cause exit(EXIT_FAILURE) eventually. */
        }

        flex_extend(source, o); /* (Which shrinks the source store) */

        /* Interpret the codes returned by HtmlIdentify. */

        switch (code)
        {
          #ifndef STRICT_PARSER

            /* Moved permanently - drop through to redirection code. Only */
            /* do this if not in a STRICT_PARSER build, as in the latter  */
            /* case slightly broken links (missing '/'s off the ends, and */
            /* so-on) aren't hidden by the browser moving on.             */

            case 301:

            /* Redirect to a GET request; this means 'definitely don't  */
            /* repost POST forms data' unlike 302 where you're supposed */
            /* to but we don't because Navigator doesn't and some sites */
            /* expect this broken behaviour (see below). So we can just */
            /* drop through to the 302 case.                            */

            case 303:

          #endif

          /* Redirect; 'redirect' is a pointer to a new URL. */

          case 302:
          {
            /* Stop the current fetch and free the source store,   */
            /* remembering to invalidate the anchor pointing to it */

            url_stop(0, handle);

            #ifdef TRACE
              if (tl & (1u<<12)) Printf("html_get_next_token: (2) flex_free block %p which held page source\n",source);
              flexcount -= flex_size(source);
              if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
            #endif

            flex_free(source);
            *source = NULL;

            /* Ensure POST requests are now cleared (we shouldn't continue */
            /* POSTing to redirected URLs - er, because Netscape Navigator */
            /* doesn't, even though we technically should.)                */

            if (up->method == URL_Method_http_POST)
            {
              /* When the headers are built, the POST data (starting with a Content-Type */
              /* entry) is put in first and everything else is inserted above it. As the */
              /* comments on this code say, this is organised so we can simplify things  */
              /* here and just chop off everything at Content-Type and below, rather     */
              /* than having to carefully remove the appropriate header lines and body.  */

              if (up->extradata)
              {
                int    len;
                int    oldbudge;
                char * strip;


                oldbudge = flex_set_budge(0);

                #ifdef DUMP_HEADERS
                  {
                    FILE * file;

                    file = fopen("<Wimp$ScrapDir>.Headers", "ab");

                    if (file)
                    {
                      fprintf(file, "Redirection from POST\r\nThere is this extra data before stripping POST-specific info:\r\n\r\n[%s]\r\n\r\n", up->extradata);

                      fclose(file);
                    }
                  }
                #endif

                strip = strstr(up->extradata, "Content-Type: ");

                if (strip)
                {
                  /* How much do we want to keep? */

                  len = strip - up->extradata + 1; /* + 1 so we can put in a string terminator where the 'C' was */

                  #ifdef TRACE
                    {
                      int rmv = flex_size((flex_ptr) &up->extradata) - len;
                      flexcount -= rmv;
                      if (tl & (1u<<13)) Printf("**   flexcount: %d\n",flexcount);
                    }
                  #endif

                  /* Resize the block */

                  flex_extend((flex_ptr) &up->extradata, len);

                  *strip = 0;
                }

                #ifdef DUMP_HEADERS
                  {
                    FILE * file;

                    file = fopen("<Wimp$ScrapDir>.Headers", "ab");

                    if (file)
                    {
                      fprintf(file, "Redirection from POST\r\nThere is this extra data after stripping POST-specific info:\r\n\r\n[%s]\r\n\r\n", up->extradata);

                      fclose(file);
                    }
                  }
                #endif

                flex_set_budge(oldbudge);
              }

              /* Change the fetch method to GET */

              up->method = URL_Method_http_GET;
            }

            /* Customer specific */

            #ifdef CUSTOMER_SPECIAL

              if (
                   !strcmp(redirect,"http://www.customer.com/login.html")  ||
                   !strcmp(redirect,"http://www.customer.com/index.html")
                 )
              {
                // Send out cookie...

                redirect = (char *) "http://www.customer.com/simple.html";
              }

            #endif

            /* Set the fetch's urlstat structure to say that */
            /* no data has been fetched                      */

            up->fetched = 0;

            /* Start a fetch on the new URL */

            e = url_get_url(URL_GetURL_AgentGiven, /* Use a custom User Agent string      */
                            handle,                /* Session handle                      */
                            up->method,            /* Fetch method                        */
                            redirect,              /* URL to get                          */
                            &up->extradata,        /* Extra data for POST etc.            */
                            NULL,                  /* We're ignoring the returned status  */
                            2);                    /* Mode 2 = fetch both header and data */

            /* Return any errors that url_get_url generated */

            if (e) return e;

            /* This function returns the address of the new URL in   */
            /* 'remaining', flagging this with a waiting status of 2 */
            /* - and yes, this is quite odd.                         */

            *waiting   = 2;
            *remaining = (int) redirect; /* Not redirect_to, as it may be freed now */
          }
          break;

          /* Authorise; the server requested authorisation before it */
          /* would deliver the page.                                 */

          case 401:
          {
            char * realm;
            char   host     [Limits_HostName];
            char   username [Limits_AuthUserWrit];
            char   password [Limits_AuthPassWrit];
            int    po;

            /* Try to find the host and realm */

            urlutils_host_name_from_url(ref_url, host, sizeof(host));

            /* (The realm will lie in the string pointed to by */
            /* 'redirect', between two double quotes).         */

            realm = authorise_read_realm(redirect);

            /* If we've already tried this, then the authorisation failed, */
            /* so display whatever authorisation failure page the server   */
            /* sent with the 401 response.                                 */

            if (up->authorised >= 2)
            {
              authorise_forget(host, realm);

              erb.errnum = Utils_Error_Custom_Message;

              StrNCpy0(erb.errmess,
                       lookup_token("BadAuthor:Authorisation failed; you must use a valid user name and password.",
                                    0,
                                    0));

              return &erb;
            }

            /* Ditch any document data got so far, we don't need it now */
            /* (it only contains e.g. header information).              */

            if (source)
            {
              flex_free((flex_ptr) source);
              *source = NULL;
            }

            up->fetched = 0;

            /* Stop the URL module trying to get anything else, and set */
            /* the flag to say we're authorising this fetch.            */

            url_stop(0, handle);
            up->authorised = 1;

            /* If there is a user name and / or password available already, */
            /* use that and authenticate immediately.                       */

            *username = *password = 0;

            po = authorise_find_user_name(host, realm);

            if (po >= 0)
            {
              StrNCpy0(username, authorise + po);

              po = authorise_find_password(host, realm);

              if (po >= 0)
              {
                StrNCpy0(password, authorise + po);
              }

              fetch_authorisation_proceed(b, up, realm, url);
            }

            /* Otherwise, get this information from a dialogue box and */
            /* allow the authentication to happen later, when the user */
            /* has done relevant things with the dialogue.             */

            else
            {
              char     prompt[Limits_AuthPrompt];
              int      f, is_ftp;
              ObjectId dbox;

              /* Ensure the authorisation dialogue is created and event handlers */
              /* are registered for it.                                          */

              e = authorise_create_dialogue((void *) b, &dbox);
              if (e) return e;

              /* Is this an FTP fetch? */

              if (!strncmp(browser_fetch_url(b), FTPmethod, strlen(FTPmethod))) is_ftp = 1;
              else                                                              is_ftp = 0;

              /* -4 corrects for %s being replaced by host / realm strings, plus */
              /* a terminator at the end of the whole lot.                       */

              f = strlen(realm) + strlen(host); /* (But no terminators needed for these, so no '+ 1's) */

              if (!is_ftp) lookup_token("Authorise:Please enter a user name and a password for %%s at %%s.",0,0);
              else         lookup_token("AuthorFTP:Please enter a password for %%s.",0,0);

              f += ((signed int) strlen(tokens)) - 4 + 1; /* Minus 4 for two lots of '%s', plus 1 for terminator */

              /* If the string is too big for the prompt or null, put a */
              /* simple version in instead.                             */

              if (f <= 0 || f > sizeof(prompt))
              {
                if (is_ftp) lookup_token("AuthorFSh:Please enter a password.",0,0);
                else        lookup_token("AuthorShr:Please enter a user name and a password.",0,0);

                e = button_set_value(0, dbox, AuthPrompt, tokens);
              }
              else
              {
                if (is_ftp) sprintf(prompt, tokens, host);
                else        sprintf(prompt, tokens, realm, host);

                e = button_set_value(0, dbox, AuthPrompt, prompt);
              }

              if (e) return e;

              /* Show the dialogue */

              RetError(toolbox_show_object(Toolbox_ShowObject_AsMenu,
                                           dbox,
                                           Toolbox_ShowObject_Centre,
                                           NULL,
                                           b->self_id,
                                           -1));

              /* If using FTP, put the user name in the writable field */
              /* and grey it out. Else just empty the field.           */

              if (is_ftp)
              {
                /* Let errors happen silently here */

                writablefield_set_value(0,
                                        dbox,
                                        AuthUserWrit,
                                        realm);

                set_gadget_state(dbox, AuthUserWrit, 1);
              }
              else e = writablefield_set_value(0, dbox, AuthUserWrit, "");

              if (e) return e;

              /* Empty the password writable */

              RetError(writablefield_set_value(0, dbox, AuthPassWrit, ""));

              /*set the authorising flag */

              menusrc = Menu_Authorise, authorising = 1;
            }
          }
          break;

          /* Catch anything else 'just in case'. Guess that the data */
          /* is parseable and let this drop through to the ordinary  */
          /* URL handling code.                                      */

          default: parseable = TYPE_HTMLFILE; /* ...so no 'break' */

          /* An ordinary URL. */

          case 200:
          {
            /* If the urlstat structure says not to parse this data, flag it as */
            /* an unknown type to prevent parsing.                              */

            if (!up->allowparse) parseable = TYPE_UNKNOWN;

            b->page_is_text = 0;

            if (b->save_link)
            {
              /* If saving out the link, flag as not parseable whatever HTMLLib said */

              *waiting = 3;
            }
            else
            {
              /* If the data is apparently parseable, flag the status as 'waiting' */
              /* else flag it as 'not parseable'.                                  */

              switch (parseable)
              {
                default:
                case TYPE_UNKNOWN:
                case TYPE_IMAGEFILE: *waiting = 3;
                break;

                /* For a text file, check the filename extension. It may */
                /* be that the server is sending garbage Content-Type    */
                /* headers, and we'll choose the filename extension over */
                /* this (initially for "http://www.batalarms.co.uk/").   */
                /*                                                       */
                /* If the server says something is HTML, we'll trust it. */

                case TYPE_TEXTFILE:
                {
                  type = urlutils_filetype_from_url(browser_fetch_url(b));

                  if (type == FileType_TEXT)
                  {
                    b->page_is_text = 1;
                    *waiting        = 1;
                  }
                  else if (type == FileType_HTML)
                  {
                    parseable = TYPE_HTMLFILE;
                    *waiting  = 1;
                  }
                  else
                  {
                    parseable = TYPE_UNKNOWN,
                    *waiting  = 3;
                  }
                }
                break;

                case TYPE_HTMLFILE: *waiting = 1;
                break;
              }
            }

            if (parseable == TYPE_IMAGEFILE && !b->save_link)
            {
              /* For images, stop the current fetch and 'redirect' to */
              /* an internal page which will fetch the image inline.  */
              /* This is inefficient as you start to fetch the image  */
              /* twice; on slow servers, something of a killer...     */
              /* Unfortunately, pressure of time (yet *again*)        */
              /* precludes a more elegant solution for the moment.    */

              url_stop(0, handle);

              #ifdef TRACE
                if (tl & (1u<<12)) Printf("html_get_next_token: (2) flex_free block %p which held page source\n",source);
                flexcount -= flex_size(source);
                if (tl & (1u<<14)) Printf("**   flexcount: %d\n",flexcount);
              #endif

              flex_free(source);
              *source = NULL;

              /* Set the fetch's urlstat structure to say that */
              /* no data has been fetched                      */

              up->fetched = 0;

              /* Want to only offer a 'back to previous page' link if you can indeed go back. */
              /* Because this page can itself add to the history, we shouldn't check the      */
              /* 'history_can_go_backwards' function result as this may return 0, even though */
              /* by the time this new page has fetched you *can* go back. So check whether or */
              /* not the local history is empty. However, if you've stepped back to the front */
              /* of the history, then you shouldn't give a 'back' link... So need to also see */
              /* if, should history_can_go_backwards return 0, history_can_go_forwards does   */
              /* as well.                                                                     */

              if (
                   !history_empty(b) &&
                   (
                     history_can_go_backwards(b) ||
                     (
                       !history_can_go_backwards(b) &&
                       !history_can_go_forwards(b)
                     )
                   )
                 )
                 redirect = Internal_URL ForExternalHImage; /* Have a 'Go back' link */

              else redirect = Internal_URL ForExternalNImage; /* No 'Go back' link     */

              /* Start a fetch on the new URL */

              e = url_get_url(URL_GetURL_AgentGiven, /* Use a custom User Agent string      */
                              handle,                /* Session handle                      */
                              up->method,            /* Fetch method                        */
                              redirect,              /* URL to get                          */
                              &up->extradata,        /* Extra data for POST etc.            */
                              NULL,                  /* We're ignoring the returned status  */
                              2);                    /* Mode 2 = fetch both header and data */

              if (e) return e;

              b->displayed = Display_External_Image;

              /* This function returns the address of the new URL in   */
              /* 'remaining', flagging this with a waiting status of 2 */
              /* - and yes, this is quite odd.                         */

              *waiting   = 2;
              *remaining = (int) redirect;
            }
            else
            {
              *remaining = type;

              /* Set the type according to the parseable flag */

              up->type = parseable;

              /* Flag that we've identified the stream */

              up->identified = 1;
            }
          }
          break;

          /* If 0, haven't identified the stream yet */

          case 0:
          break;

        /* Closure of 'switch' statement checking the return code */
        /* from the HtmlIdentify call                             */
        }

      /* Closure of 'if' statement that checked there was recognised data */
      /* following HTTP style headers in the data stream                  */
      }

    /* Closure of 'if' statement that checked the urlstat structure  */
    /* to see if authorisation was in progress (and only proceeded   */
    /* if it was not)                                                */
    }

  /* Closure of 'if' statement that followed the url_read_data    */
  /* function call and associated memory allocation procedures,   */
  /* and only proceeded if data had been fetched and authoristion */
  /* was not flagged as being in progress.                        */
  }

  /* If we're not in the middle of authorisation, there hasn't */
  /* been an error, and the urlstat structure for the fetch    */
  /* flags that fetching is still in progress, ask the URL     */
  /* module if things are finished yet. If so, do the relevant */
  /* tidying up and unset the 'fetch in progress' flag.        */

  if (!e && up->authorised != 1 && !authorising && up->fetching)
  {
    int s;

    if (image || b->displayed == Display_Fetched_Page || b->save_link)
    {
      e = url_status(0, handle, &s, NULL, NULL);
      if (e) return e;

      if (s & URL_Status_Done)
      {
        up->fetching = 0;
        *waiting     = 0;
      }
    }
  }

  /* If we've been passed somewhere to put the size of the store, */
  /* and if the store is present, return the size of it.          */

  if (size) *size = (*source) ? flex_size(source) : 0;

  #ifdef TRACE
    if (tl & (1u<<6))
    {
      if (!e) Printf("html_get_next_token: Successful\n");
      else Printf("html_get_next_token: Exitting with an error\n");
    }
  #endif

  /* Exit, passing on any error if there is one */

  return (e);
}