listserv.yml 8 KB
Newer Older
Wágner Ferenc's avatar
Wágner Ferenc committed
1
2
3
4
5
6
---
- hosts: lilac.mail.einfra.hu
  gather_subset: min

  roles:
  # Exim first, otherwise debian-basic pulls in sendmail
7
  - { role: mailman, tags: mailman }
Wágner Ferenc's avatar
Wágner Ferenc committed
8
9
10
11
12
13
  - common.debian-basic
  - common.monitored-server
  - common.munin-node
  - common.nevtar
  - common.sudoers
  - common.persistent-journal
14
15
  - { role: common.metricbeat,  tags: common.metricbeat }
  - { role: common.journalbeat, tags: common.journalbeat }
16
17
18
  # The apache role must come late because it
  # * shares the certificate key with the Debian-exim group and
  # * configures Munin plugins.  And role dependencies lead to repeats.
19
  - { role: apache, tags: apache }
20
  - { role: common.nftables, tags: common.nftables }
Wágner Ferenc's avatar
Wágner Ferenc committed
21
22
23

  vars:
  - nss_cn: listserv.niif.hu
24
  - sudoers: "{{ sudoers_default|union(['aviktor']) }}"
25
  - elastic_template_name: mail
Wágner Ferenc's avatar
Wágner Ferenc committed
26
  - list_domain: listserv.niif.hu
27
28
  - munin_extra_packages:
    - munin-plugins-apache
29
  - munin_custom_plugins:
30
31
32
33
      apache_accesses: apache_accesses
      apache_volume: apache_volume
      apache_process_detail: apache_process_detail
      entropy: entropy
34
35
      exim_mailstats: exim_mailstats
      exim_mailqueue: exim_mailqueue
36
37
38
  - nrpe_custom_checks:
      check_mail_queue: /usr/lib/nagios/plugins/check_frozen
      check_Mailman: /usr/bin/sudo -u list /usr/lib/nagios/plugins/check_mailman
39
40
41
42
43
44
45
46
  - accept_tcp:
    - ssh
    - https
    - smtp
    - submission
  - reject_tcp:
    - auth
    - http
Wágner Ferenc's avatar
Wágner Ferenc committed
47
48
49
50
51
52
53
54
55
56

  tasks:
  # This enables TAB completion of host names
  - name: Disable hashing of known hosts by ssh
    become: yes
    lineinfile:
      dest: /etc/ssh/ssh_config
      regexp: ^[# \t]*HashKnownHosts[ ]
      line: "    HashKnownHosts no"
    tags: ssh
57

58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
  - name: Install Mailman-specific monitoring plugins
    become: yes
    apt:
      name: monitoring-plugins-mailman
    tags: nrpe

  - name: Let NRPE daemon run the Mailman check as user list
    become: yes
    copy:
      content: |
        nagios ALL = (list) NOPASSWD: /usr/lib/nagios/plugins/check_mailman
      dest: /etc/sudoers.d/60_nagios_ansible
      owner: root
      group: root
      mode: 0440
    tags: nrpe

75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
# Then:
# - add IPv6 entry to /etc/hosts
# - set up filesystem for /var/lib/mailman, preferably before installation,
#   but at least do a systemd daemon-load to enroll it under local-fs.target,
#   otherwise it gets umounted early and Mailman can't be stopped:
#   Traceback (most recent call last):
#     File "/var/lib/mailman/bin/qrunner", line 278, in <module>
#        main()
#     File "/var/lib/mailman/bin/qrunner", line 238, in main
#        qrunner.run()
#     File "/var/lib/mailman/Mailman/Queue/Runner.py", line 70, in run
#     File "/var/lib/mailman/Mailman/Queue/Runner.py", line 94, in _oneloop
#     File "/var/lib/mailman/Mailman/Queue/Switchboard.py", line 194, in files
#   OSError :  [Errno 2] No such file or directory: '/var/lib/mailman/qfiles/commands' 
# - dpkg-reconfigure mailman, select en,hu (based on current statistics, these are enough)
# - sudo adduser wferi list
# - /var/lib/mailman/bin/newlist mailman, then as in the old wiki:
#   - Advertise this list when people ask what lists are on this machine? No
#   - Who can view subscription list? List members (probably default)
#   - Is archive file source for public or private archival? private
# - sudo systemctl start mailman
# - mmsitepass (creates /var/lib/mailman/data/adm.pw)

# Migration of the Test list:
# - stop exim4, apache2 and mailman services, disable /etc/cron.d/mailman (or do this in the afternoon)
100
# - wferi@listserv2:/var/lib/mailman$ tar -cvzf /tmp/test.tgz archives/private/test archives/private/test.mbox archives/public/test lists/test
101
102
103
# - wferi@lilac:/var/lib/mailman$ sudo tar -xvf ~/test.tgz
# - no archive regeneration to keep the URLs (in case of past mbox modifications) and to reduce migration load

104
105
106
107
108
109
110
111
112
113
114
115
116
# Planned migration of all lists except test and mailman (run in a screen session):
# - wferi@lilac:~$ sudo rsync -vaz --super --exclude /lists/test --exclude /lists/mailman --exclude /archives/private/test --exclude /archives/private/test.mbox --exclude /archives/private/mailman --exclude /archives/private/mailman.mbox listserv2.niif.hu:/var/lib/mailman/archives :/var/lib/mailman/data :/var/lib/mailman/lists /var/lib/mailman 2>&1 | tee >(gzip --stdout > rsync.log.gz)
# rsync+sshd saturates CPU on listserv2?
# sent 60,490,702 bytes  received 99,447,023,926 bytes  2,841,975.66 bytes/sec
# total size is 181,666,346,642  speedup is 1.83
# rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1677) [generator=3.1.3]
# repeat from at 23:04, result:
# -rw-rw-r-- 1 wferi wferi 11525 Jul  8 00:11 rsync.log.2.gz
# sent 3,267,532 bytes  received 63,007,941 bytes  16,513.14 bytes/sec
# total size is 181,679,920,306  speedup is 2,741.28
# rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1677) [generator=3.1.3]
# (probably fixed by making nep.mbox.extra readable)

117
118
119
120
121
# Note:
# - the list of lists depends on the used HTTP domain (internal virtual hosting)
# - hbone-ticketing@listserv.niif.hu sender whitelist makes no sense (wiki change 32 and 33), omitted

# TODO
122
123
# - check out https://www.msapiro.net/scripts/
# - route local mail to redirector
124
125
126
127
# - Why does this show the mailman role twice? Dependency infelicity...
#   ansible-playbook -i inventory listserv.yml -K --check --diff
# - check that outgoing traffic isn't sent to rspamd
# - Namazu
128
# - test@lista.edu.hu? Működik. Érdemes lenne spéci routert csinálni erre a domainre is?
129
130
131
132
133
134
135
136
137
138
# - install auth (ident) daemon
# - skip or fail on rspamd failure?
#   2021-06-09 12:29:36 1lqvSZ-0000oZ-9V spam acl condition: spamd: failed to connect to any address for rspamd.mail.einfra.hu: Connection timed out
#   2021-06-09 12:29:36 1lqvSZ-0000oZ-9V H=noc6.vh.hbone.hu (noc6) [2001:738:0:1:214:22ff:fe13:e172] Warning: ACL "warn" statement skipped: condition test deferred
# - why set MAIN_TLS_VERIFY_CERTIFICATES to /dev/null like listserv2?
# - check freeze_tell setting (does it work?)
# - check mailman-* forwarding to postmaster (me)
# - why was the system_aliases router moved forward?
# - DEFAULT_CHARSET for the archives
# - MX changes: listserv.niif.hu, ipv6forum.hu
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178

# Takeover:
# wferi@lilac:~$ sudoedit /etc/apache2/apache2.conf # catch-all
# wferi@lilac:~$ sudo systemctl reload apache2.service
# wferi@listserv2:~$ sudo service exim4 stop
# [sudo] password for wferi:
# [ ok ] Stopping MTA: exim4_listener.
# wferi@listserv2:~$ sudo service apache2 stop
# [ ok ] Stopping web server: apache2 ... waiting .
# wferi@listserv2:~$ sudo service mailman stop
# [ ok ] Stopping Mailman master qrunner: mailmanctl.
# wferi@listserv2:/etc/cron.d$ sudo mv mailman mailman.disabled
# wferi@lilac:~$ sudo systemctl stop exim4.service
# wferi@lilac:~$ sudo systemctl stop mailman.service
# wferi@pdns1:~$ pdnsutil edit-zone niif.hu
# [...]
# -niif.hu 86400 IN SOA ns2.iif.hu hostmaster.iif.hu 2021070501 43200 7200 172800 3600
# +niif.hu 86400 IN SOA ns2.iif.hu hostmaster.iif.hu 2021071301 43200 7200 172800 3600
# -listserv.niif.hu 300 IN A 193.225.14.155
# +listserv.niif.hu 300 IN A 195.111.92.17
# -listserv.niif.hu 300 IN MX 10 listserv2.niif.hu
# +listserv.niif.hu 300 IN MX 10 lilac.mail.einfra.hu
# -listserv.niif.hu 300 IN AAAA 2001:738:0:701:216:3eff:fe01:0
# +listserv.niif.hu 300 IN AAAA 2001:738:0:415::6
#
# RSYNC
#
# wferi@lilac:~$ sudoedit /etc/apache2/apache2.conf # remove catch-all
# wferi@lilac:~$ sudo systemctl reload apache2.service
# wferi@lilac:~$ sudo systemctl start mailman.service
# wferi@lilac:~$ sudo systemctl start exim4.service
#
# PROBLEMS
#
# listserv2 /etc/aliases contains stuff
#   - abuse list is no more
#
# Apache inconsistencies - why not the same size?
# 193.6.168.233 - - [13/Jul/2021:17:26:52 +0200] "GET /mailman/admindb/fekosz HTTP/1.1" 200 5312 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
# 193.6.168.233 - - [13/Jul/2021:17:27:04 +0200] "GET /mailman/admindb/elnokseg HTTP/1.1" 200 1214 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"