69 lines
3.4 KiB
C
69 lines
3.4 KiB
C
/*************************************************************************/
|
|
/* */
|
|
/* Language Technologies Institute */
|
|
/* Carnegie Mellon University */
|
|
/* Copyright (c) 1999 */
|
|
/* All Rights Reserved. */
|
|
/* */
|
|
/* Permission is hereby granted, free of charge, to use and distribute */
|
|
/* this software and its documentation without restriction, including */
|
|
/* without limitation the rights to use, copy, modify, merge, publish, */
|
|
/* distribute, sublicense, and/or sell copies of this work, and to */
|
|
/* permit persons to whom this work is furnished to do so, subject to */
|
|
/* the following conditions: */
|
|
/* 1. The code must retain the above copyright notice, this list of */
|
|
/* conditions and the following disclaimer. */
|
|
/* 2. Any modifications must be clearly marked as such. */
|
|
/* 3. Original authors' names are not deleted. */
|
|
/* 4. The authors' names are not used to endorse or promote products */
|
|
/* derived from this software without specific prior written */
|
|
/* permission. */
|
|
/* */
|
|
/* CARNEGIE MELLON UNIVERSITY AND THE CONTRIBUTORS TO THIS WORK */
|
|
/* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
|
|
/* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
|
|
/* SHALL CARNEGIE MELLON UNIVERSITY NOR THE CONTRIBUTORS BE LIABLE */
|
|
/* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
|
|
/* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
|
|
/* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
|
|
/* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
|
|
/* THIS SOFTWARE. */
|
|
/* */
|
|
/*************************************************************************/
|
|
/* Author: Alan W Black (awb@cs.cmu.edu) */
|
|
/* Date: August 1999 */
|
|
/*************************************************************************/
|
|
/* */
|
|
/* test for tokenizer of strings and files */
|
|
/* */
|
|
/*************************************************************************/
|
|
#include <stdio.h>
|
|
#include "cst_tokenstream.h"
|
|
|
|
int main(int argc, char **argv)
|
|
{
|
|
/* need that cool argument parser */
|
|
cst_tokenstream *fd;
|
|
const char *token;
|
|
|
|
fd = ts_open("data.one");
|
|
|
|
while (!ts_eof(fd))
|
|
{
|
|
token = ts_get(fd);
|
|
|
|
printf("ws >%s<\n",fd->whitespace);
|
|
printf("pp >%s<\n",fd->prepunctuation);
|
|
printf("tk >%s<\n",token);
|
|
printf("pp >%s<\n",fd->postpunctuation);
|
|
printf("fp >%d<\n",fd->file_pos);
|
|
printf("ln >%d<\n",fd->line_number);
|
|
printf("\n");
|
|
|
|
}
|
|
|
|
ts_close(fd);
|
|
|
|
return 0;
|
|
}
|