tests: add scripting API tests (issue #104)

Automatic tests of scripting API are made with Python scripts:

- unparse.py: convert Python code to other languages
- testapigen.py: generate scripts in all languages to test the API
- testapi.py scripting API tests

.travis.yml

@@ -31,6 +31,9 @@ script:
     - msgcheck po/*.po
     - pylint --version
     - pylint doc/docgen.py
+    - pylint tests/scripts/python/testapigen.py
+    - pylint tests/scripts/python/testapi.py
+    - pylint tests/scripts/python/unparse.py
 
 after_success:
     - weechat --help

ChangeLog.adoc

@@ -49,7 +49,8 @@ Bug fixes::
 
 Tests::
 
-  * display an error if the required locale en_US.UTF-8 is not installed
+  * scripts: add scripting API tests (issue #104)
+  * unit: display an error if the required locale en_US.UTF-8 is not installed
 
 [[v1.9.1]]
 == Version 1.9.1 (2017-09-23)
@@ -330,8 +331,8 @@ Documentation::
 
 Tests::
 
-  * add a test to check if all plugins are loaded
+  * unit: add a test to check if all plugins are loaded
-  * fix locale used to execute tests (issue #631)
+  * unit: fix locale used to execute tests (issue #631)
 
 Build::
 
@@ -564,7 +565,7 @@ Bug fixes::
 
 Tests::
 
-  * fix memory leak in tests launcher
+  * unit: fix memory leak in tests launcher
 
 Build::
 
@@ -735,7 +736,7 @@ Build::
 
 Tests::
 
-  * add unit tests using CppUTest
+  * unit: add unit tests using CppUTest (issue #104)
 
 [[v0.4.3]]
 == Version 0.4.3 (2014-02-09)

doc/en/weechat_dev.en.adoc

@@ -88,6 +88,8 @@ The main WeeChat directories are:
 |       trigger/    | Trigger plugin.
 |       xfer/       | Xfer plugin (IRC DCC file/chat).
 | tests/            | Tests.
+|    scripts/       | Scripting API tests.
+|       python/     | Python scripts to generate and run the scripting API tests.
 |    unit/          | Unit tests.
 |       core/       | Unit tests for core functions.
 | doc/              | Documentation.
@@ -339,7 +341,13 @@ WeeChat "core" is located in following directories:
 |===
 | Path/file                   | Description
 | tests/                      | Root of tests.
-|    tests.cpp                | Program used to run tests.
+|    tests.cpp                | Program used to run all tests.
+|    scripts/                 | Root of scripting API tests.
+|       test-scripts.cpp      | Program used to run the scripting API tests.
+|       python/               | Python scripts to generate and run the scripting API tests.
+|          testapigen.py      | Python script generating scripts in all languages to test the scripting API.
+|          testapi.py         | Python script with scripting API tests, used by script testapigen.py.
+|          unparse.py         | Convert Python code to other languages, used by script testapigen.py.
 |    unit/                    | Root of unit tests.
 |       core/                 | Root of unit tests for core.
 |          test-arraylist.cpp | Tests: arraylists.

doc/fr/weechat_dev.fr.adoc

@@ -90,6 +90,8 @@ Les répertoires principaux de WeeChat sont :
 |       trigger/    | Extension Trigger.
 |       xfer/       | Extension Xfer (IRC DCC fichier/discussion).
 | tests/            | Tests.
+|    scripts/       | Tests de l'API script.
+|       python/     | Scripts Python pour générer et lancer les tests de l'API script.
 |    unit/          | Tests unitaires.
 |       core/       | Tests unitaires pour les fonctions du cœur.
 | doc/              | Documentation.
@@ -341,7 +343,13 @@ Le cœur de WeeChat est situé dans les répertoires suivants :
 |===
 | Chemin/fichier              | Description
 | tests/                      | Racine des tests.
-|    tests.cpp                | Programme utilisé pour lancer les tests.
+|    tests.cpp                | Programme utilisé pour lancer tous les tests.
+|    scripts/                 | Racine des tests de l'API script.
+|       test-scripts.cpp      | Programme utilisé pour lancer les tests de l'API script.
+|       python/               | Scripts Python pour générer et lancer les tests de l'API script.
+|          testapigen.py      | Script Python générant des scripts dans tous les languages pour tester l'API script.
+|          testapi.py         | Script Python avec les tests API, utilisé par le script testapigen.py.
+|          unparse.py         | Conversion de code Python vers d'autres langages, utilisé par le script testapigen.py.
 |    unit/                    | Racine des tests unitaires.
 |       core/                 | Racine des tests unitaires pour le cœur.
 |          test-arraylist.cpp | Tests : listes avec tableau (« arraylists »).

doc/ja/weechat_dev.ja.adoc

@@ -94,6 +94,9 @@ qweechat::
 |       trigger/    | trigger プラグイン
 |       xfer/       | xfer (IRC DCC ファイル/チャット)
 | tests/            | テスト
+// TRANSLATION MISSING
+|    scripts/       | Scripting API tests.
+|       python/     | Python scripts to generate and run the scripting API tests.
 |    unit/          | 単体テスト
 |       core/       | コア関数の単体テスト
 | doc/              | 文書
@@ -345,7 +348,14 @@ WeeChat "core" は以下のディレクトリに配置されています:
 |===
 | パス/ファイル名             | 説明
 | tests/                      | テスト用のルートディレクトリ
-|    tests.cpp                | テスト実行に使うプログラム
+// TRANSLATION MISSING
+|    tests.cpp                | Program used to run all tests.
+|    scripts/                 | Root of scripting API tests.
+|       test-scripts.cpp      | Program used to run the scripting API tests.
+|       python/               | Python scripts to generate and run the scripting API tests.
+|          testapigen.py      | Python script generating scripts in all languages to test the scripting API.
+|          testapi.py         | Python script with scripting API tests, used by script testapigen.py.
+|          unparse.py         | Convert Python code to other languages, used by script testapigen.py.
 |    unit/                    | 単体テスト用のルートディレクトリ
 |       core/                 | core 向け単体テスト用のルートディレクトリ
 |          test-arraylist.cpp | テスト: 配列リスト

tests/CMakeLists.txt

@@ -39,6 +39,7 @@ set(LIB_WEECHAT_UNIT_TESTS_SRC
   unit/core/test-url.cpp
   unit/core/test-utf8.cpp
   unit/core/test-util.cpp
+  scripts/test-scripts.cpp
 )
 add_library(weechat_unit_tests STATIC ${LIB_WEECHAT_UNIT_TESTS_SRC})
 
@@ -77,4 +78,5 @@ add_test(NAME unit
   COMMAND tests -v)
 set_property(TEST unit PROPERTY
   ENVIRONMENT "WEECHAT_TESTS_ARGS=-p;"
-  "WEECHAT_EXTRA_LIBDIR=${PROJECT_BINARY_DIR}/src")
+  "WEECHAT_EXTRA_LIBDIR=${PROJECT_BINARY_DIR}/src;"
+  "WEECHAT_TESTS_SCRIPTS_DIR=${CMAKE_CURRENT_SOURCE_DIR}/scripts/python")

tests/scripts/python/testapi.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017 Sébastien Helleu <flashcode@flashtux.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+This script contains WeeChat scripting API tests
+(it can not be run directly and can not be loaded in WeeChat).
+
+It is parsed by testapigen.py, using Python AST (Abstract Syntax Trees),
+to generate scripts in all supported languages (Python, Perl, Ruby, ...).
+The resulting scripts can be loaded in WeeChat to test the scripting API.
+"""
+
+# pylint: disable=line-too-long,no-value-for-parameter
+
+import weechat  # pylint: disable=import-error
+
+
+def check(result, condition, lineno):
+    """Display the result of a test."""
+    if result:
+        weechat.prnt('', '      TEST OK: ' + condition)
+    else:
+        weechat.prnt('',
+                     'SCRIPT_SOURCE' + ':' + lineno + ':1: ' +
+                     'ERROR: [' + 'SCRIPT_NAME' + '] condition is false: ' +
+                     condition)
+
+
+def test_plugins():
+    """Test plugins functions."""
+    check(weechat.plugin_get_name('') == 'core')
+    check(weechat.plugin_get_name(weechat.buffer_get_pointer(weechat.buffer_search_main(), 'plugin')) == 'core')
+
+
+def test_strings():
+    """Test string functions."""
+    check(weechat.charset_set('iso-8859-15') == 1)
+    check(weechat.charset_set('') == 1)
+    check(weechat.iconv_to_internal('iso-8859-15', 'abc') == 'abc')
+    check(weechat.iconv_from_internal('iso-8859-15', 'abcd') == 'abcd')
+    check(weechat.gettext('abcdef') == 'abcdef')
+    check(weechat.ngettext('file', 'files', 1) == 'file')
+    check(weechat.ngettext('file', 'files', 2) == 'files')
+    check(weechat.strlen_screen('abcd') == 4)
+    check(weechat.string_match('abcdef', 'abc*', 0) == 1)
+    check(weechat.string_eval_path_home('test ${abc}', {}, {'abc': '123'}, {}) == 'test 123')
+    check(weechat.string_mask_to_regex('test*mask') == 'test.*mask')
+    check(weechat.string_has_highlight('my test string', 'test,word2') == 1)
+    check(weechat.string_has_highlight_regex('my test string', 'test|word2') == 1)
+    check(weechat.string_remove_color('test', '?') == 'test')
+    check(weechat.string_is_command_char('/test') == 1)
+    check(weechat.string_is_command_char('test') == 0)
+    check(weechat.string_input_for_buffer('test') == 'test')
+    check(weechat.string_input_for_buffer('/test') == '')
+    check(weechat.string_input_for_buffer('//test') == '/test')
+    check(weechat.string_eval_expression("100 > 50", {}, {}, {"type": "condition"}) == '1')
+    check(weechat.string_eval_expression("${buffer.full_name}", {}, {}, {}) == 'core.weechat')
+
+
+def test_lists():
+    """Test list functions."""
+    ptr_list = weechat.list_new()
+    check(ptr_list != '')
+    check(weechat.list_size(ptr_list) == 0)
+    item_def = weechat.list_add(ptr_list, 'def', weechat.WEECHAT_LIST_POS_SORT, '')
+    check(weechat.list_size(ptr_list) == 1)
+    item_abc = weechat.list_add(ptr_list, 'abc', weechat.WEECHAT_LIST_POS_SORT, '')
+    check(weechat.list_size(ptr_list) == 2)
+    check(weechat.list_search(ptr_list, 'abc') == item_abc)
+    check(weechat.list_search(ptr_list, 'def') == item_def)
+    check(weechat.list_search(ptr_list, 'ghi') == '')
+    check(weechat.list_search_pos(ptr_list, 'abc') == 0)
+    check(weechat.list_search_pos(ptr_list, 'def') == 1)
+    check(weechat.list_search_pos(ptr_list, 'ghi') == -1)
+    check(weechat.list_casesearch(ptr_list, 'abc') == item_abc)
+    check(weechat.list_casesearch(ptr_list, 'def') == item_def)
+    check(weechat.list_casesearch(ptr_list, 'ghi') == '')
+    check(weechat.list_casesearch(ptr_list, 'ABC') == item_abc)
+    check(weechat.list_casesearch(ptr_list, 'DEF') == item_def)
+    check(weechat.list_casesearch(ptr_list, 'GHI') == '')
+    check(weechat.list_casesearch_pos(ptr_list, 'abc') == 0)
+    check(weechat.list_casesearch_pos(ptr_list, 'def') == 1)
+    check(weechat.list_casesearch_pos(ptr_list, 'ghi') == -1)
+    check(weechat.list_casesearch_pos(ptr_list, 'ABC') == 0)
+    check(weechat.list_casesearch_pos(ptr_list, 'DEF') == 1)
+    check(weechat.list_casesearch_pos(ptr_list, 'GHI') == -1)
+    check(weechat.list_get(ptr_list, 0) == item_abc)
+    check(weechat.list_get(ptr_list, 1) == item_def)
+    check(weechat.list_get(ptr_list, 2) == '')
+    weechat.list_set(item_def, 'def2')
+    check(weechat.list_string(item_def) == 'def2')
+    check(weechat.list_next(item_abc) == item_def)
+    check(weechat.list_next(item_def) == '')
+    check(weechat.list_prev(item_abc) == '')
+    check(weechat.list_prev(item_def) == item_abc)
+    weechat.list_remove(ptr_list, item_abc)
+    check(weechat.list_size(ptr_list) == 1)
+    check(weechat.list_get(ptr_list, 0) == item_def)
+    check(weechat.list_get(ptr_list, 1) == '')
+    weechat.list_remove_all(ptr_list)
+    check(weechat.list_size(ptr_list) == 0)
+    weechat.list_free(ptr_list)
+
+
+def weechat_init():
+    """Main function."""
+    weechat.register('SCRIPT_NAME', 'SCRIPT_AUTHOR', 'SCRIPT_VERSION',
+                     'SCRIPT_LICENSE', 'SCRIPT_DESCRIPTION', '', '')
+    weechat.prnt('', '>>>')
+    weechat.prnt('', '>>> ------------------------------')
+    weechat.prnt('', '>>> Testing ' + 'SCRIPT_LANGUAGE' + ' API')
+    weechat.prnt('', '  > TESTS: ' + 'SCRIPT_TESTS')
+    test_plugins()
+    test_strings()
+    test_lists()
+    weechat.prnt('', '  > TESTS END')

tests/scripts/python/testapigen.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017 Sébastien Helleu <flashcode@flashtux.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""
+Scripts generator for WeeChat: build source of scripts in all languages to
+test the scripting API.
+
+This script can be run in WeeChat or as a standalone script
+(during automatic tests, it is loaded as a WeeChat script).
+
+It uses the following scripts:
+- unparse.py: convert Python code to other languages (including Python itself)
+- testapi.py: the WeeChat scripting API tests
+"""
+
+from __future__ import print_function
+import argparse
+import ast
+from datetime import datetime
+import inspect
+try:
+    from StringIO import StringIO  # python 2
+except ImportError:
+    from io import StringIO  # python 3
+import os
+import sys
+import traceback
+
+sys.dont_write_bytecode = True
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(SCRIPT_DIR)
+from unparse import (  # pylint: disable=wrong-import-position
+    UnparsePython,
+    UnparsePerl,
+    UnparseRuby,
+    UnparseLua,
+    UnparseTcl,
+    UnparseGuile,
+    UnparseJavascript,
+    UnparsePhp,
+)
+
+RUNNING_IN_WEECHAT = True
+try:
+    import weechat
+except ImportError:
+    RUNNING_IN_WEECHAT = False
+
+
+SCRIPT_NAME = 'testapigen'
+SCRIPT_AUTHOR = 'Sébastien Helleu <flashcode@flashtux.org>'
+SCRIPT_VERSION = '0.1'
+SCRIPT_LICENSE = 'GPL3'
+SCRIPT_DESC = 'Generate scripting API test scripts'
+
+SCRIPT_COMMAND = 'testapigen'
+
+
+class WeechatScript(object):  # pylint: disable=too-many-instance-attributes
+    """
+    A generic WeeChat script.
+
+    This class must NOT be instanciated directly, use subclasses instead:
+    PythonScript, PerlScript, ...
+    """
+
+    def __init__(self, unparse_class, tree, source_script, output_dir,
+                 language, extension, comment_char='#',
+                 weechat_module='weechat'):
+        # pylint: disable=too-many-arguments
+        self.unparse_class = unparse_class
+        self.tree = tree
+        self.source_script = os.path.realpath(source_script)
+        self.output_dir = os.path.realpath(output_dir)
+        self.language = language
+        self.extension = extension
+        self.script_name = 'testapi.%s' % extension
+        self.script_path = os.path.join(self.output_dir, self.script_name)
+        self.comment_char = comment_char
+        self.weechat_module = weechat_module
+        self.rename_functions()
+        self.replace_variables()
+
+    def comment(self, string):
+        """Get a commented line."""
+        return '%s %s' % (self.comment_char, string)
+
+    def rename_functions(self):
+        """Rename some API functions in the tree."""
+        functions = {
+            'prnt': 'print',
+            'prnt_date_tags': 'print_date_tags',
+            'prnt_y': 'print_y',
+        }
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Call) and \
+                    isinstance(node.func, ast.Attribute) and \
+                    node.func.value.id == 'weechat':
+                node.func.attr = functions.get(node.func.attr, node.func.attr)
+
+    def replace_variables(self):
+        """Replace script variables in string values."""
+        variables = {
+            'SCRIPT_SOURCE': self.source_script,
+            'SCRIPT_NAME': self.script_name,
+            'SCRIPT_PATH': self.script_path,
+            'SCRIPT_AUTHOR': 'Sebastien Helleu',
+            'SCRIPT_VERSION': '1.0',
+            'SCRIPT_LICENSE': 'GPL3',
+            'SCRIPT_DESCRIPTION': ('%s scripting API test' %
+                                   self.language.capitalize()),
+            'SCRIPT_LANGUAGE': self.language,
+        }
+        # count the total number of tests
+        tests_count = 0
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Call) and \
+                    isinstance(node.func, ast.Name) and \
+                    node.func.id == 'check':
+                tests_count += 1
+        variables['SCRIPT_TESTS'] = str(tests_count)
+        # replace variables
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Str) and \
+                    node.s in variables:
+                node.s = variables[node.s]
+
+    def write_header(self, output):
+        """Generate script header (just comments by default)."""
+        comments = (
+            '',
+            '%s -- WeeChat %s scripting API testing' % (
+                self.script_name, self.language.capitalize()),
+            '',
+            'WeeChat script automatically generated by testapigen.py.',
+            'DO NOT EDIT BY HAND!',
+            '',
+            'Date: %s' % datetime.now(),
+            '',
+        )
+        for line in comments:
+            output.write(self.comment(line).rstrip() + '\n')
+
+    def write(self):
+        """Write script on disk."""
+        print('Writing script %s... ' % self.script_path, end='')
+        with open(self.script_path, 'w') as output:
+            self.write_header(output)
+            self.unparse_class(output).add(self.tree)
+            output.write('\n')
+            self.write_footer(output)
+        print('OK')
+
+    def write_footer(self, output):
+        """Write footer (nothing by default)."""
+        pass
+
+
+class WeechatPythonScript(WeechatScript):
+    """A WeeChat script written in Python."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatPythonScript, self).__init__(
+            UnparsePython, tree, source_script, output_dir, 'python', 'py')
+
+    def rename_functions(self):
+        # nothing to rename in Python
+        pass
+
+    def write_header(self, output):
+        output.write('# -*- coding: utf-8 -*-\n')
+        super(WeechatPythonScript, self).write_header(output)
+        output.write('\n'
+                     'import weechat')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     '\n'
+                     'if __name__ == "__main__":\n'
+                     '    weechat_init()\n')
+
+
+class WeechatPerlScript(WeechatScript):
+    """A WeeChat script written in Perl."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatPerlScript, self).__init__(
+            UnparsePerl, tree, source_script, output_dir, 'perl', 'pl')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     'weechat_init();\n')
+
+
+class WeechatRubyScript(WeechatScript):
+    """A WeeChat script written in Ruby."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatRubyScript, self).__init__(
+            UnparseRuby, tree, source_script, output_dir, 'ruby', 'rb')
+
+    def rename_functions(self):
+        super(WeechatRubyScript, self).rename_functions()
+        for node in ast.walk(self.tree):
+            if isinstance(node, ast.Attribute) and \
+                    node.value.id == 'weechat':
+                node.value.id = 'Weechat'
+
+
+class WeechatLuaScript(WeechatScript):
+    """A WeeChat script written in Lua."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatLuaScript, self).__init__(
+            UnparseLua, tree, source_script, output_dir, 'lua', 'lua',
+            comment_char='--')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     'weechat_init()\n')
+
+
+class WeechatTclScript(WeechatScript):
+    """A WeeChat script written in Tcl."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatTclScript, self).__init__(
+            UnparseTcl, tree, source_script, output_dir, 'tcl', 'tcl')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     'weechat_init\n')
+
+
+class WeechatGuileScript(WeechatScript):
+    """A WeeChat script written in Guile (Scheme)."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatGuileScript, self).__init__(
+            UnparseGuile, tree, source_script, output_dir, 'guile', 'scm',
+            comment_char=';')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     '(weechat_init)\n')
+
+
+class WeechatJavascriptScript(WeechatScript):
+    """A WeeChat script written in Javascript."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatJavascriptScript, self).__init__(
+            UnparseJavascript, tree, source_script, output_dir,
+            'javascript', 'js', comment_char='//')
+
+    def write_footer(self, output):
+        output.write('\n'
+                     'weechat_init()\n')
+
+
+class WeechatPhpScript(WeechatScript):
+    """A WeeChat script written in PHP."""
+
+    def __init__(self, tree, source_script, output_dir):
+        super(WeechatPhpScript, self).__init__(
+            UnparsePhp, tree, source_script, output_dir, 'php', 'php',
+            comment_char='//')
+
+    def write_header(self, output):
+        output.write('<?php\n')
+        super(WeechatPhpScript, self).write_header(output)
+
+    def write_footer(self, output):
+        output.write('\n'
+                     'weechat_init();\n')
+
+# ============================================================================
+
+
+def update_nodes(tree):
+    """
+    Update the tests AST tree (in-place):
+      1. add a print message in each test_* function
+      2. add arguments in calls to check() function
+    """
+    for node in ast.walk(tree):
+        if isinstance(node, ast.FunctionDef) and \
+                node.name.startswith('test_'):
+            # add a print at the beginning of each test function
+            node.body.insert(
+                0, ast.parse('weechat.prnt("", "  > %s");' % node.name))
+        elif isinstance(node, ast.Call) and \
+                isinstance(node.func, ast.Name) and \
+                node.func.id == 'check':
+            # add two arguments in the call to "check" function:
+            #   1. the string representation of the test
+            #   2. the line number in source (as string)
+            # for example if this test is on line 50:
+            #   check(weechat.test() == 123)
+            # it becomes:
+            #   check(weechat.test() == 123, 'weechat.test() == 123', '50')
+            output = StringIO()
+            unparsed = UnparsePython(output=output)
+            unparsed.add(node.args[0])
+            node.args.append(ast.Str(output.getvalue()))
+            node.args.append(ast.Str(str(node.func.lineno)))
+
+
+def get_tests(path):
+    """Parse the source with tests and return the AST node."""
+    test_script = open(path).read()
+    tests = ast.parse(test_script)
+    update_nodes(tests)
+    return tests
+
+
+def generate_scripts(source_script, output_dir):
+    """Generate scripts in all languages to test the API."""
+    ret_code = 0
+    error = None
+    try:
+        for name, obj in inspect.getmembers(sys.modules[__name__]):
+            if inspect.isclass(obj) and name != 'WeechatScript' and \
+                    name.startswith('Weechat') and name.endswith('Script'):
+                tests = get_tests(source_script)
+                obj(tests, source_script, output_dir).write()
+    except Exception as exc:  # pylint: disable=broad-except
+        ret_code = 1
+        error = 'ERROR: %s\n\n%s' % (str(exc), traceback.format_exc())
+    return ret_code, error
+
+
+def testapigen_cmd_cb(data, buf, args):
+    """Callback for WeeChat command /testapigen."""
+
+    def print_error(msg):
+        """Print an error message on core buffer."""
+        weechat.prnt('', '%s%s' % (weechat.prefix('error'), msg))
+
+    try:
+        source_script, output_dir = args.split()
+    except ValueError:
+        print_error('ERROR: invalid arguments for /testapigen')
+        return weechat.WEECHAT_RC_OK
+    if not weechat.mkdir_parents(output_dir, 0o755):
+        print_error('ERROR: invalid directory: %s' % output_dir)
+        return weechat.WEECHAT_RC_OK
+    ret_code, error = generate_scripts(source_script, output_dir)
+    if error:
+        print_error(error)
+    return weechat.WEECHAT_RC_OK if ret_code == 0 else weechat.WEECHAT_RC_ERROR
+
+
+def get_parser_args():
+    """Get parser arguments."""
+    parser = argparse.ArgumentParser(
+        description=('Generate WeeChat scripts in all languages '
+                     'to test the API.'))
+    parser.add_argument(
+        'script',
+        help='the path to Python script with tests')
+    parser.add_argument(
+        '-o', '--output-dir',
+        default='.',
+        help='output directory (defaults to current directory)')
+    return parser.parse_args()
+
+
+def main():
+    """Main function (when script is not loaded in WeeChat)."""
+    args = get_parser_args()
+    ret_code, error = generate_scripts(args.script, args.output_dir)
+    if error:
+        print(error)
+    sys.exit(ret_code)
+
+
+if __name__ == '__main__':
+    if RUNNING_IN_WEECHAT:
+        weechat.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION,
+                         SCRIPT_LICENSE, SCRIPT_DESC, '', '')
+        weechat.hook_command(
+            SCRIPT_COMMAND,
+            'Generate scripting API test scripts',
+            'source_script output_dir',
+            'source_script: path to source script (testapi.py)\n'
+            '   output_dir: output directory for scripts',
+            '',
+            'testapigen_cmd_cb', '')
+    else:
+        main()

tests/scripts/test-scripts.cpp

@@ -0,0 +1,243 @@
+/*
+ * test-scripts.cpp - test scripting API
+ *
+ * Copyright (C) 2017 Sébastien Helleu <flashcode@flashtux.org>
+ *
+ * This file is part of WeeChat, the extensible chat client.
+ *
+ * WeeChat is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * WeeChat is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with WeeChat.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "CppUTest/TestHarness.h"
+
+extern "C"
+{
+#ifndef HAVE_CONFIG_H
+#define HAVE_CONFIG_H
+#endif
+#include <stdio.h>
+#include <string.h>
+#include "src/core/weechat.h"
+#include "src/core/wee-string.h"
+#include "src/core/wee-hook.h"
+#include "src/plugins/plugin.h"
+}
+
+extern void run_cmd (const char *command);
+
+struct t_hook *api_hook_print = NULL;
+int api_tests_ok = 0;
+int api_tests_errors = 0;
+int api_tests_count = 0;
+int api_tests_end = 0;
+int api_tests_other = 0;
+
+
+TEST_GROUP(Scripts)
+{
+    /*
+     * Callback for any message displayed by WeeChat or a plugin.
+     */
+
+    static int
+    test_print_cb (const void *pointer, void *data, struct t_gui_buffer *buffer,
+                   time_t date, int tags_count, const char **tags, int displayed,
+                   int highlight, const char *prefix, const char *message)
+    {
+        const char *pos;
+        char *error;
+        int value;
+
+        /* make C++ compiler happy */
+        (void) pointer;
+        (void) data;
+        (void) buffer;
+        (void) date;
+        (void) tags_count;
+        (void) tags;
+        (void) displayed;
+        (void) highlight;
+        (void) prefix;
+
+        if (message)
+        {
+            pos = strstr (message, "> TESTS: ");
+            if (pos)
+            {
+                error = NULL;
+                value = (int)strtol (pos + 9, &error, 10);
+                if (error && !error[0])
+                    api_tests_count = value;
+            }
+            else if (strstr (message, "TEST OK"))
+                api_tests_ok++;
+            else if (strstr (message, "ERROR"))
+                api_tests_errors++;
+            else if (strstr (message, "TESTS END"))
+                api_tests_end++;
+            else if ((message[0] != '>') && (message[0] != ' '))
+                api_tests_other++;
+        }
+
+        return WEECHAT_RC_OK;
+    }
+
+    void setup()
+    {
+        /*
+         * TODO: fix memory leaks in javascript plugin
+         * and remove this function
+         */
+        MemoryLeakWarningPlugin::turnOffNewDeleteOverloads();
+
+        api_hook_print = hook_print (NULL,  /* plugin */
+                                     NULL,  /* buffer */
+                                     NULL,  /* tags */
+                                     NULL,  /* message */
+                                     1,     /* strip colors */
+                                     &test_print_cb,
+                                     NULL,
+                                     NULL);
+    }
+
+    void teardown()
+    {
+        unhook (api_hook_print);
+
+        /*
+         * TODO: fix memory leaks in javascript plugin
+         * and remove this function
+         */
+        MemoryLeakWarningPlugin::turnOnNewDeleteOverloads();
+    }
+};
+
+/*
+ * Tests scripting API.
+ */
+
+TEST(Scripts, API)
+{
+    char path_testapigen[PATH_MAX], path_testapi[PATH_MAX];
+    char *path_testapi_output_dir, str_command[4096];
+    const char *languages[][2] = {
+        { "python",    "py"  },
+        { "perl",      "pl"  },
+        { "ruby",      "rb"  },
+        { "lua",       "lua" },
+        { "tcl",       "tcl" },
+        { "scm",       "scm" },
+        { "jvascript", "js"  },
+        { "php",       "php" },
+        { NULL,        NULL  }
+    };
+    int i;
+
+    printf ("...\n");
+
+    /* build paths for scripting API tests */
+    snprintf (path_testapigen, sizeof (path_testapigen),
+              "%s%s%s",
+              getenv ("WEECHAT_TESTS_SCRIPTS_DIR"),
+              DIR_SEPARATOR,
+              "testapigen.py");
+    snprintf (path_testapi, sizeof (path_testapi),
+              "%s%s%s",
+              getenv ("WEECHAT_TESTS_SCRIPTS_DIR"),
+              DIR_SEPARATOR,
+              "testapi.py");
+    path_testapi_output_dir = string_eval_path_home ("%h/testapi",
+                                                     NULL, NULL, NULL);
+    CHECK(path_testapi_output_dir);
+
+    api_tests_ok = 0;
+    api_tests_errors = 0;
+
+    /* load generator script */
+    snprintf (str_command, sizeof (str_command),
+              "/script load %s", path_testapigen);
+    run_cmd (str_command);
+
+    /* generate scripts to test API */
+    snprintf (str_command, sizeof (str_command),
+              "/testapigen %s %s",
+              path_testapi,
+              path_testapi_output_dir);
+    run_cmd (str_command);
+
+    /* check that there was no errors in script generation */
+    LONGS_EQUAL(0, api_tests_errors);
+
+    /* unload generator scritp */
+    snprintf (str_command, sizeof (str_command),
+              "/script unload testapigen.py");
+    run_cmd (str_command);
+
+    /* test the scripting API */
+    for (i = 0; languages[i][0]; i++)
+    {
+        api_tests_ok = 0;
+        api_tests_errors = 0;
+        api_tests_count = 0;
+        api_tests_end = 0;
+        api_tests_other = 0;
+
+        /* load script (run tests) */
+        snprintf (str_command, sizeof (str_command),
+                  "/script load -q %s/testapi.%s",
+                  path_testapi_output_dir,
+                  languages[i][1]);
+        run_cmd (str_command);
+
+        /* display results */
+        printf ("\n");
+        printf (">>> Tests %s: %d tests, %d OK, %d errors, "
+                "%d unexpected messages\n",
+                languages[i][0],
+                api_tests_count,
+                api_tests_ok,
+                api_tests_errors,
+                api_tests_other);
+        printf ("\n");
+
+        /* unload script */
+        snprintf (str_command, sizeof (str_command),
+                  "/script unload -q testapi.%s",
+                  languages[i][1]);
+        run_cmd (str_command);
+
+        /* check that tests were found in script */
+        CHECK(api_tests_count > 0);
+
+        /* check that all tests are OK */
+        LONGS_EQUAL(api_tests_count, api_tests_ok);
+
+        /* check that there was no errors */
+        LONGS_EQUAL(0, api_tests_errors);
+
+        /* check that end of script was reached (no syntax error) */
+        LONGS_EQUAL(1, api_tests_end);
+
+        /*
+         * check that there was no warning/error from plugin
+         * (if everything is OK, there are 2 messages when the script is loaded
+         * and 2 messages when it is unloaded, so total is 4)
+         */
+        LONGS_EQUAL(0, api_tests_other);
+    }
+
+    free (path_testapi_output_dir);
+
+    printf ("TEST(Scripts, API)");
+}

tests/tests.cpp

@@ -36,6 +36,7 @@ extern "C"
 #include "src/core/wee-hook.h"
 #include "src/core/wee-input.h"
 #include "src/core/wee-string.h"
+#include "src/core/wee-util.h"
 #include "src/plugins/plugin.h"
 #include "src/gui/gui-main.h"
 #include "src/gui/gui-buffer.h"
@@ -49,6 +50,8 @@ extern "C"
 
 #define LOCALE_TESTS "en_US.UTF-8"
 
+#define WEECHAT_TESTS_HOME "./tmp_weechat_test"
+
 /* import tests from libs */
 IMPORT_TEST_GROUP(Plugins);
 IMPORT_TEST_GROUP(Arraylist);
@@ -61,7 +64,23 @@ IMPORT_TEST_GROUP(String);
 IMPORT_TEST_GROUP(Url);
 IMPORT_TEST_GROUP(Utf8);
 IMPORT_TEST_GROUP(Util);
+IMPORT_TEST_GROUP(Scripts);
 
+struct t_gui_buffer *ptr_core_buffer = NULL;
+
+
+/*
+ * Callback for exec_on_files (to remove all files in WeeChat home directory).
+ */
+
+void
+exec_on_files_cb (void *data, const char *filename)
+{
+    /* make C++ compiler happy */
+    (void) data;
+
+    unlink (filename);
+}
 
 /*
  * Callback for any message displayed by WeeChat or a plugin.
@@ -69,10 +88,8 @@ IMPORT_TEST_GROUP(Util);
 
 int
 test_print_cb (const void *pointer, void *data, struct t_gui_buffer *buffer,
-               time_t date, int tags_count,
+               time_t date, int tags_count, const char **tags, int displayed,
-               const char **tags, int displayed,
+               int highlight, const char *prefix, const char *message)
-               int highlight, const char *prefix,
-               const char *message)
 {
     /* make C++ compiler happy */
     (void) pointer;
@@ -119,6 +136,17 @@ test_gui_init ()
     gui_main_init ();
 }
 
+/*
+ * Displays and runs a command on a buffer.
+ */
+
+void
+run_cmd (const char *command)
+{
+    printf (">>> Running command: %s\n", command);
+    input_data (ptr_core_buffer, command);
+}
+
 /*
  * Runs tests in WeeChat environment.
  */
@@ -128,7 +156,6 @@ main (int argc, char *argv[])
 {
     int rc, length, weechat_argc;
     char *weechat_tests_args, *args, **weechat_argv;
-    struct t_gui_buffer *ptr_core_buffer;
 
     /* setup environment: English language, no specific timezone */
     setenv ("LC_ALL", LOCALE_TESTS, 1);
@@ -144,6 +171,9 @@ main (int argc, char *argv[])
         return 1;
     }
 
+    /* clean WeeChat home */
+    util_exec_on_files (WEECHAT_TESTS_HOME, 1, 1, &exec_on_files_cb, NULL);
+
     /* build arguments for WeeChat */
     weechat_tests_args = getenv ("WEECHAT_TESTS_ARGS");
     length = strlen (argv[0]) +
@@ -157,8 +187,9 @@ main (int argc, char *argv[])
         return 1;
     }
     snprintf (args, length,
-              "%s --dir ./tmp_weechat_test%s%s",
+              "%s --dir %s%s%s",
               argv[0],
+              WEECHAT_TESTS_HOME,
               (weechat_tests_args) ? " " : "",
               (weechat_tests_args) ? weechat_tests_args : "");
     weechat_argv = string_split_shell (args, &weechat_argc);
@@ -184,9 +215,9 @@ main (int argc, char *argv[])
     }
 
     /* display WeeChat version and directories */
-    input_data (ptr_core_buffer, "/command core version");
+    run_cmd ("/command core version");
-    input_data (ptr_core_buffer, "/debug dirs");
+    run_cmd ("/debug dirs");
-    input_data (ptr_core_buffer, "/debug libs");
+    run_cmd ("/debug libs");
 
     /* run all tests */
     printf ("\n");