annotate upstream_least_conn.t @ 1585:bff287fbf347

Tests: added proxy_cookie_domain/path tests with "off" parameter.
author Sergey Kandaurov <pluknet@nginx.com>
date Thu, 23 Jul 2020 12:17:39 +0300
parents 882267679006
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
1 #!/usr/bin/perl
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
2
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
3 # (C) Maxim Dounin
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
4
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
5 # Tests for upstream least_conn balancer module.
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
6
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
7 ###############################################################################
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
8
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
9 use warnings;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
10 use strict;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
11
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
12 use Test::More;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
13
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
14 BEGIN { use FindBin; chdir($FindBin::Bin); }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
15
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
16 use lib 'lib';
438
60888e2c3f5a Tests: new http_start() and http_end() functions.
Sergey Kandaurov <pluknet@nginx.com>
parents: 400
diff changeset
17 use Test::Nginx qw/ :DEFAULT http_end /;
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
18
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
19 ###############################################################################
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
20
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
21 select STDERR; $| = 1;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
22 select STDOUT; $| = 1;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
23
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
24 my $t = Test::Nginx->new()->has(qw/http proxy upstream_least_conn/)->plan(2);
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
25
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
26 $t->write_file_expand('nginx.conf', <<'EOF');
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
27
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
28 %%TEST_GLOBALS%%
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
29
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
30 daemon off;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
31
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
32 events {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
33 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
34
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
35 http {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
36 %%TEST_GLOBALS_HTTP%%
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
37
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
38 upstream u {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
39 least_conn;
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
40 server 127.0.0.1:8081;
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
41 server 127.0.0.1:8082;
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
42 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
43
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
44 server {
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
45 listen 127.0.0.1:8080;
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
46 server_name localhost;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
47
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
48 location / {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
49 proxy_pass http://u;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
50 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
51 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
52 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
53
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
54 EOF
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
55
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
56 $t->run_daemon(\&http_daemon, port(8081));
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
57 $t->run_daemon(\&http_daemon, port(8082));
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
58 $t->run();
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
59
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
60 $t->waitforsocket('127.0.0.1:' . port(8081));
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
61 $t->waitforsocket('127.0.0.1:' . port(8082));
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
62
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
63 ###############################################################################
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
64
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
65 my @ports = my ($port1, $port2) = (port(8081), port(8082));
952
e9064d691790 Tests: converted tests to run in parallel.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 549
diff changeset
66
e9064d691790 Tests: converted tests to run in parallel.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 549
diff changeset
67 is(many('/', 10), "$port1: 5, $port2: 5", 'balanced');
549
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
68
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
69 my @sockets;
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
70 push(@sockets, http_get('/w', start => 1));
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
71 push(@sockets, http_get('/w', start => 1));
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
72
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
73 select undef, undef, undef, 0.2;
e4ff43e00d84 Tests: refactored least_conn test.
Sergey Kandaurov <pluknet@nginx.com>
parents: 532
diff changeset
74
952
e9064d691790 Tests: converted tests to run in parallel.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 549
diff changeset
75 is(many('/w', 10), "$port2: 10", 'least conn');
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
76
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
77 ###############################################################################
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
78
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
79 sub many {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
80 my ($uri, $count) = @_;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
81 my %ports;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
82
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
83 for (1 .. $count) {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
84 if (http_get($uri) =~ /X-Port: (\d+)/) {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
85 $ports{$1} = 0 unless defined $ports{$1};
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
86 $ports{$1}++;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
87 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
88 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
89
952
e9064d691790 Tests: converted tests to run in parallel.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 549
diff changeset
90 my @keys = map { my $p = $_; grep { $p == $_ } keys %ports } @ports;
e9064d691790 Tests: converted tests to run in parallel.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 549
diff changeset
91 return join ', ', map { $_ . ": " . $ports{$_} } @keys;
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
92 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
93
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
94 ###############################################################################
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
95
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
96 sub http_daemon {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
97 my ($port) = @_;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
98
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
99 my $server = IO::Socket::INET->new(
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
100 Proto => 'tcp',
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
101 LocalHost => '127.0.0.1',
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
102 LocalPort => $port,
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
103 Listen => 5,
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
104 Reuse => 1
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
105 )
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
106 or die "Can't create listening socket: $!\n";
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
107
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
108 local $SIG{PIPE} = 'IGNORE';
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
109
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
110 while (my $client = $server->accept()) {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
111 $client->autoflush(1);
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
112
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
113 my $headers = '';
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
114 my $uri = '';
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
115
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
116 while (<$client>) {
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
117 $headers .= $_;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
118 last if (/^\x0d?\x0a?$/);
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
119 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
120
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
121 $uri = $1 if $headers =~ /^\S+\s+([^ ]+)\s+HTTP/i;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
122
974
882267679006 Tests: simplified parallel modifications in tests.
Andrey Zelenkov <zelenkov@nginx.com>
parents: 952
diff changeset
123 if ($uri eq '/w' && $port == port(8081)) {
400
98d9b06b087b Tests: adjust least_conn test timeouts.
Maxim Dounin <mdounin@mdounin.ru>
parents: 342
diff changeset
124 Test::Nginx::log_core('||', "$port: sleep(2.5)");
98d9b06b087b Tests: adjust least_conn test timeouts.
Maxim Dounin <mdounin@mdounin.ru>
parents: 342
diff changeset
125 select undef, undef, undef, 2.5;
294
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
126 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
127
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
128 Test::Nginx::log_core('||', "$port: response, 200");
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
129 print $client <<EOF;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
130 HTTP/1.1 200 OK
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
131 Connection: close
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
132 X-Port: $port
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
133
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
134 OK
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
135 EOF
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
136
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
137 close $client;
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
138 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
139 }
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
140
71e275487aeb Tests: upstream least_conn and ip_hash tests.
Maxim Dounin <mdounin@mdounin.ru>
parents:
diff changeset
141 ###############################################################################