Scanner Implementation.

convert Prolog text into tokens More...

Detailed Description

convert Prolog text into tokens

The tokens are supposed to be used by the parser, but can also be exported as a list

Description:

This module produces a list of tokens for use by the parser The calling program should supply a routine int nextch(charpos) int *charpos; which, when called should produce the next char or -1 if none availlable The scanner will stop producing tokens when it either finds an end of file (-1) or a token consisting of just '' followed by a blank or control char Scanner errors will be signalled by the scanner exiting with a non- zero ErrorMsg and ErrorPos Note that, even in this case, the scanner will try to find the end of the term A function char *AllocScannerMemory(nbytes) should be supplied for allocating (temporary) space for strings and for the table of prolog variables occurring in the term

---

Class Documentation

scanner_extra_alloc

struct scanner_extra_alloc

Definition at line 187 of file scanner.c.

Public Attributes
struct scanner_extra_alloc *	next
void *	filler

Member Data Documentation

filler

void* filler

Definition at line 189 of file scanner.c.

next

struct scanner_extra_alloc* next

Definition at line 188 of file scanner.c.

Macro Definition Documentation

add_ch_to_buff

#define add_ch_to_buff

(

ch

)

Value:

  {\
  if (ch == 10 && (trueGlobalPrologFlag(ISO_FLAG) ||            \
           falseLocalPrologFlag(MULTILINE_QUOTED_TEXT_FLAG)))\
 {  \
      /* in ISO a new line terminates a string */                              \
            }\
      if (t) { t->Tok = Ord(kind = eot_tok);                                            \
 }    \
  charp += put_utf8(charp, ch); }

Definition at line 884 of file scanner.c.

CHECK_SPACE

#define CHECK_SPACE

(

)

Value:

  if (ASP - HR < 1024) {                                                       \
    LOCAL_ErrorMessage = "Stack Overflow";                                     \
    LOCAL_Error_TYPE = RESOURCE_ERROR_STACK;                                   \
    LOCAL_Error_Size = 0L;                                                     \
    if (p) {                                                                   \
      p->Tok = Ord(kind = eot_tok);                                            \
      p->TokInfo = TermOutOfStackError;                                        \
    }                                                                          \
    /* serious error now */                                                    \
    return l;                                                                  \
  }

Definition at line 731 of file scanner.c.

CodeSpaceError

#define CodeSpaceError	(		t,
			p,
			l
	)		CodeSpaceError__(t, p, l PASS_REGS)

Definition at line 192 of file scanner.c.

my_islower

#define my_islower

(

C

)

(C >= 'a' && C <= 'z')

Definition at line 77 of file scanner.c.

my_isupper

#define my_isupper

(

C

)

(C >= 'A' && C <= 'Z')

Definition at line 76 of file scanner.c.

my_isxdigit

#define my_isxdigit	(		C,
			SU,
			SL
	)		(chtype(C) == NU || (C >= 'A' && C <= (SU)) || (C >= 'a' && C <= (SL)))

Definition at line 74 of file scanner.c.

number_overflow

#define number_overflow

(

)

Value:

  {                                                                            \
    imgsz = Yap_Min(imgsz * 2, imgsz);                                         \
    char *nbuf;                                                                \
    nbuf = Realloc(buf, imgsz);                                                \
    left = imgsz - max_size;                                                   \
    max_size = imgsz;                                                          \
    buf = nbuf;                                                                \
  }

Definition at line 435 of file scanner.c.

safe_add_ch_to_buff

#define safe_add_ch_to_buff

(

ch

)

charp += put_utf8(charp, ch);

Definition at line 882 of file scanner.c.

TrailSpaceError

#define TrailSpaceError	(		t,
			l
	)		TrailSpaceError__(t, l PASS_REGS)

Definition at line 205 of file scanner.c.

Function Documentation

Yap_clean_tokenizer()

void Yap_clean_tokenizer

(

void

)

terminate scanning: just closes the comment store

Definition at line 1577 of file scanner.c.

Yap_scan_num()

Term Yap_scan_num

(

StreamDesc *

inp

)

This routine is used when we need to parse a string into a number.

Definition at line 666 of file scanner.c.

Yap_tokenizer()

TokEntry * Yap_tokenizer	(	void *	st_,
		void *	params_
	)

Definition at line 895 of file scanner.c.

Yap_tokRep()

Term Yap_tokRep

(

void *

tokptre

)

convert a token to text

represent token *_tokptr_ in string s, maxlength is sz-1

Definition at line 747 of file scanner.c.

Yap_tokText()

const char * Yap_tokText

(

void *

tokptre

)

Definition at line 792 of file scanner.c.

Variable Documentation

Yap_chtype0

char_kind_t Yap_chtype0[NUMBER_OF_CHARS+1]

Definition at line 103 of file scanner.c.

Macros
#define	my_isxdigit(C, SU, SL)    (chtype(C) == NU || (C >= 'A' && C <= (SU)) || (C >= 'a' && C <= (SL)))
#define	my_isupper(C)   (C >= 'A' && C <= 'Z')
#define	my_islower(C)   (C >= 'a' && C <= 'z')
#define	CodeSpaceError(t, p, l)   CodeSpaceError__(t, p, l PASS_REGS)
#define	TrailSpaceError(t, l)   TrailSpaceError__(t, l PASS_REGS)
#define	number_overflow()
#define	CHECK_SPACE()
#define	safe_add_ch_to_buff(ch)   charp += put_utf8(charp, ch);
#define	add_ch_to_buff(ch)

Typedefs
typedef struct scanner_extra_alloc	ScannerExtraBlock

Functions
double	atof (const char *)
Term	Yap_scan_num (StreamDesc *inp)
	This routine is used when we need to parse a string into a number. More...
Term	Yap_tokRep (void *tokptre)
	convert a token to text More...
const char *	Yap_tokText (void *tokptre)
TokEntry *	Yap_tokenizer (void *st_, void *params_)
void	Yap_clean_tokenizer (void)
	terminate scanning: just closes the comment store More...

Variables
char_kind_t	Yap_chtype0 [NUMBER_OF_CHARS+1]